{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "# 选择评价指标为accuracy，毕竟只有0或1\n",
    "#SVM并不能直接输出各类的概率，所以在这个例子中我们用正确率作为模型预测性能的度量\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = ''\n",
    "# './data/'\n",
    "data = pd.read_csv(dpath +\"diabetes.csv\")\n",
    "data.head()\n",
    "data_row =len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>120.894531</td>\n",
       "      <td>69.105469</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>31.992578</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>31.972618</td>\n",
       "      <td>19.355807</td>\n",
       "      <td>15.952218</td>\n",
       "      <td>115.244002</td>\n",
       "      <td>7.884160</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   768.000000  768.000000     768.000000     768.000000  768.000000   \n",
       "mean      3.845052  120.894531      69.105469      20.536458   79.799479   \n",
       "std       3.369578   31.972618      19.355807      15.952218  115.244002   \n",
       "min       0.000000    0.000000       0.000000       0.000000    0.000000   \n",
       "25%       1.000000   99.000000      62.000000       0.000000    0.000000   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   30.500000   \n",
       "75%       6.000000  140.250000      80.000000      32.000000  127.250000   \n",
       "max      17.000000  199.000000     122.000000      99.000000  846.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age     Outcome  \n",
       "count  768.000000                768.000000  768.000000  768.000000  \n",
       "mean    31.992578                  0.471876   33.240885    0.348958  \n",
       "std      7.884160                  0.331329   11.760232    0.476951  \n",
       "min      0.000000                  0.078000   21.000000    0.000000  \n",
       "25%     27.300000                  0.243750   24.000000    0.000000  \n",
       "50%     32.000000                  0.372500   29.000000    0.000000  \n",
       "75%     36.600000                  0.626250   41.000000    1.000000  \n",
       "max     67.100000                  2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Glu, BloodPressure, skin, insulin bmi中有0值。从特征定义来看，0为缺少数据，且非正常含义需要进行数据补足分别尝试用mean和众数来补足"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>99</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.254</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.258</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0          1.0       99           70.0            0.0      0.0  32.0   \n",
       "1          NaN      100            NaN            NaN      NaN   NaN   \n",
       "\n",
       "   DiabetesPedigreeFunction   Age  Outcome  \n",
       "0                     0.254  22.0      0.0  \n",
       "1                     0.258   NaN      NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pregnancies                   3.845052\n",
       "Glucose                     120.894531\n",
       "BloodPressure                69.105469\n",
       "SkinThickness                20.536458\n",
       "Insulin                      79.799479\n",
       "BMI                          31.992578\n",
       "DiabetesPedigreeFunction      0.471876\n",
       "Age                          33.240885\n",
       "Outcome                       0.348958\n",
       "dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## glu使用均值。blood使用众数。skin使用均值，众数为0。Insulin使用均值，众数为0。BMI均值和众数相近，使用众数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>121.681605</td>\n",
       "      <td>72.295573</td>\n",
       "      <td>26.606479</td>\n",
       "      <td>118.660163</td>\n",
       "      <td>32.450911</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>30.436016</td>\n",
       "      <td>12.106756</td>\n",
       "      <td>9.631241</td>\n",
       "      <td>93.080358</td>\n",
       "      <td>6.875366</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>18.200000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.750000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>27.500000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   768.000000  768.000000     768.000000     768.000000  768.000000   \n",
       "mean      3.845052  121.681605      72.295573      26.606479  118.660163   \n",
       "std       3.369578   30.436016      12.106756       9.631241   93.080358   \n",
       "min       0.000000   44.000000      24.000000       7.000000   14.000000   \n",
       "25%       1.000000   99.750000      64.000000      20.536458   79.799479   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   79.799479   \n",
       "75%       6.000000  140.250000      80.000000      32.000000  127.250000   \n",
       "max      17.000000  199.000000     122.000000      99.000000  846.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age     Outcome  \n",
       "count  768.000000                768.000000  768.000000  768.000000  \n",
       "mean    32.450911                  0.471876   33.240885    0.348958  \n",
       "std      6.875366                  0.331329   11.760232    0.476951  \n",
       "min     18.200000                  0.078000   21.000000    0.000000  \n",
       "25%     27.500000                  0.243750   24.000000    0.000000  \n",
       "50%     32.000000                  0.372500   29.000000    0.000000  \n",
       "75%     36.600000                  0.626250   41.000000    1.000000  \n",
       "max     67.100000                  2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Glucose'] = data['Glucose'].where(data['Glucose']!=0.0, data.mean()['Glucose'])\n",
    "data['BloodPressure'] = data['BloodPressure'].where(data['BloodPressure']!=0.0, data.mode()['BloodPressure'][0])\n",
    "data['SkinThickness'] = data['SkinThickness'].where(data['SkinThickness']!=0.0, data.mean()['SkinThickness'])\n",
    "data['Insulin'] = data['Insulin'].where(data['Insulin']!=0.0, data.mean()['Insulin'])\n",
    "data['BMI'] = data['BMI'].where(data['BMI']!=0.0, data.mode()['BMI'][0])\n",
    "\n",
    "\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEy1JREFUeJzt3X3QZnV93/H3hycfUORpYVaWdXFck5BYCN1SlBkVSFLRBpBKo8MEQhk3HbEhwWlFa2VSm4kmQQya2m6CZbEGglTkIUwSsgJOrKgLCkiIZaUGtkvYNSL4EEyQb/+4fne4XM7ee5bl3Odi7/dr5prrnN/5nXN9b2flM+fh9zupKiRJ2tpuYxcgSZpNBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE57jF3AzjjwwANrxYoVY5chSc8qt9122zerasn2+j2rA2LFihWsX79+7DIk6VklyV/36eclJklSJwNCktTJgJAkdTIgJEmdDAhJUqdBAyLJN5LcleQrSda3tv2T3Jjk3va9X2tPkouTbEhyZ5KjhqxNkjS/hTiDOK6qjqyqVW39fGBdVa0E1rV1gBOBle2zGvjoAtQmSdqGMS4xnQysbctrgVOm2i+riVuBfZMsHaE+SRLDB0QBf5bktiSrW9vBVfUgQPs+qLUfAjwwte/G1iZJGsHQI6mPrapNSQ4CbkzyV/P0TUdbPaXTJGhWAyxfvnynC/yn//6ynT6Gdj23/fYZY5cgjW7QM4iq2tS+NwNXA0cDD81dOmrfm1v3jcChU7svAzZ1HHNNVa2qqlVLlmx3KhFJ0tM0WEAk2TvJC+eWgZ8DvgpcC5zZup0JXNOWrwXOaE8zHQM8MncpSpK08Ia8xHQwcHWSud/5w6r6kyRfAq5McjZwP3Ba638D8HpgA/B94KwBa5MkbcdgAVFV9wFHdLT/LXBCR3sB5wxVjyRpxziSWpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSp8EDIsnuSb6c5Pq2fliSLyS5N8kfJdmrtT+nrW9o21cMXZskadsW4gziXOCeqfUPABdV1UrgYeDs1n428HBVvQy4qPWTJI1k0IBIsgx4A/AHbT3A8cBVrcta4JS2fHJbp20/ofWXJI1g6DOIDwH/AXiirR8AfLuqHm/rG4FD2vIhwAMAbfsjrb8kaQSDBUSSfwlsrqrbpps7ulaPbdPHXZ1kfZL1W7ZseQYqlSR1GfIM4ljgpCTfAK5gcmnpQ8C+SfZofZYBm9ryRuBQgLb9RcC3tj5oVa2pqlVVtWrJkiUDli9Ji9tgAVFV76qqZVW1Angz8JmqOh24CXhT63YmcE1bvrat07Z/pqqecgYhSVoYY4yDeCdwXpINTO4xXNLaLwEOaO3nAeePUJskqdlj+112XlXdDNzclu8Dju7o8xhw2kLUI0naPkdSS5I6GRCSpE4GhCSpkwEhSeq03YBIsneS3dryy5OclGTP4UuTJI2pzxnEZ4HnJjkEWAecBVw6ZFGSpPH1CYhU1feBU4EPV9UbgcOHLUuSNLZeAZHklcDpwB+3tgUZPyFJGk+fgPhV4F3A1VV1d5KXMpkuQ5K0C9vumUBV3QLckmTvtn4f8CtDFyZJGlefp5hemeQvaW+FS3JEkv86eGWSpFH1ucT0IeBfAH8LUFV3AK8esihJ0vh6DZSrqge2avrhALVIkmZIn6eRHkjyKqCS7MXk/sM9w5YlSRpbnzOIfwucw+Sd0RuBI4G3DVmUJGl8fc4gfqy9Ce4fJTkW+NwwJUmSZkGfM4gP92yTJO1CtnkG0UZPvwpYkuS8qU37ALsPXZgkaVzzXWLaC3hB6/PCqfZHgTcNWZQkaXzbDIipEdSXVtVfJ9m7qr63gLVJkkbU5x7Eix1JLUmLjyOpJUmdHEktSerkSGpJUqcdHUn9/5iMpD5nyKIkSePr8z6IbzJ5m5wkaRHp8z6Ilya5LsmWJJuTXNPeKidJ2oX1ucT0h8CVwFLgxcAngcuHLEqSNL4+AZGq+nhVPd4+/xOooQuTJI1rvrmY9m+LNyU5H7iCSTD8AvDHC1CbJGlE892kvo1JIKSt//LUtgLeN1RRkqTxzTcX02ELWYgkabb0GShHkp8CDgeeO9dWVZcNVZQkaXx9HnO9gMkLgj4MHAf8FnBSj/2em+SLSe5IcneSX2/thyX5QpJ7k/xRG51Nkue09Q1t+4qd+LskSTupz1NMbwJOAP6mqs4CjgCe02O/HwDHV9URTEZfvy7JMcAHgIuqaiXwMHB263828HBVvQy4qPWTJI2kT0D8XVU9ATyeZB9gM7DdgXI18d22umf7FHA8cFVrXwuc0pZPbuu07SckmbtBLklaYH0CYn2SfYHfZ/Jk0+3AF/scPMnuSb7CJFRuBL4OfLuqHm9dNjKZ44n2/QBA2/4IcEDPv0OS9AzrMxfT29rif0vyJ8A+VXVnn4NX1Q+BI1vAXA38RFe39t11tvCUAXlJVgOrAZYvX96nDEnS07DNM4gkP96+j5r7APsDe7Tl3qrq28DNwDHAvknmgmkZsKktbwQObb+5B/Ai4Fsdx1pTVauqatWSJUt2pAxJ0g6Y7wziHcBbgQs7ts3dS9imJEuAf6iqbyd5HvAzTG4838TkxvcVwJnANW2Xa9v659v2z1SVU3pI0kjmGyj31vZ93NM89lJgbZLdmZypXFlV17f3W1+R5L8AXwYuaf0vAT6eZAOTM4c3P83flSQ9A+abi+nU+Xasqk9tZ/udwE93tN8HHN3R/hhw2nzHlCQtnPkuMf18+z4IeBXwmbZ+HJP7CfMGhCTp2W2+S0xnASS5Hji8qh5s60uB31uY8iRJY+kzDmLFXDg0DwEvH6geSdKM6DNZ381J/pTJW+SKyc3jmwatSpI0uj4D5d6e5I3Aq1vTmqq6etiyJElj6zXddwsEQ0GSFpE+9yAkSYuQASFJ6jTfQLl1VXVCkg9U1TsXsihJcP9/fsXYJWgGLX/vXQv2W/Pdg1ia5DXASUmuYKvZVqvq9kErkySNar6AeC9wPpMZVz+41bbtTtYnSXp2m28k9VXAVUn+U1W9bwFrkiTNgD7jIN6X5CSeHAdxc1VdP2xZkqSxbfcppiS/CZwL/GX7nNvaJEm7sD4D5d4AHFlVTwAkWcvkPQ7vGrIwSdK4+o6D2Hdq+UVDFCJJmi19ziB+E/hykpuYPOr6ajx7kKRdXp+b1JcnuRn4Z0wC4p1V9TdDFyZJGlffyfoeBK4duBZJ0gxxLiZJUicDQpLUad6ASLJbkq8uVDGSpNkxb0C0sQ93JFm+QPVIkmZEn5vUS4G7k3wR+N5cY1WdNFhVkqTR9QmIXx+8CknSzOkzDuKWJC8BVlbVnyd5PrD78KVJksbUZ7K+twJXAf+9NR0CfHrIoiRJ4+vzmOs5wLHAowBVdS9w0JBFSZLG1ycgflBVfz+3kmQPJm+UkyTtwvoExC1J3g08L8nPAp8Erhu2LEnS2PoExPnAFuAu4JeBG4D3DFmUJGl8fZ5ieqK9JOgLTC4tfa2qvMQkSbu4Pk8xvQH4OnAx8BFgQ5ITe+x3aJKbktyT5O4k57b2/ZPcmOTe9r1fa0+Si5NsSHJnkqN27k+TJO2MPpeYLgSOq6rXVtVrgOOAi3rs9zjwjqr6CeAY4JwkhzO5ZLWuqlYC69o6wInAyvZZDXx0h/4SSdIzqk9AbK6qDVPr9wGbt7dTVT1YVbe35e8A9zAZQ3EysLZ1Wwuc0pZPBi6riVuBfZMs7fdnSJKeadu8B5Hk1LZ4d5IbgCuZ3IM4DfjSjvxIkhXATzO5j3FwewERVfVgkrkxFYcAD0zttrG1PbgjvyVJembMd5P656eWHwJe05a3APv1/YEkLwD+F/CrVfVokm127Wh7ys3wJKuZXIJi+XInmZWkoWwzIKrqrJ09eJI9mYTDJ6rqU635oSRL29nDUp68XLUROHRq92XApo661gBrAFatWuXTVJI0kO0+5prkMODfASum+29vuu9MThUuAe6pqg9ObboWOBN4f/u+Zqr97UmuAP458MjcpShJ0sLrM933p5n8h/464IkdOPaxwC8CdyX5Smt7N5NguDLJ2cD9TO5pwGQA3uuBDcD3gZ0+g5EkPX19AuKxqrp4Rw9cVX9B930FgBM6+heTiQElSTOgT0D8bpILgD8DfjDXOPcIqyRp19QnIF7B5FLR8Tx5ianauiRpF9UnIN4IvHR6ym9J0q6vz0jqO4B9hy5EkjRb+pxBHAz8VZIv8aP3IOZ9zFWS9OzWJyAuGLwKSdLM6fM+iFsWohBJ0mzpM5L6Ozw5J9JewJ7A96pqnyELkySNq88ZxAun15OcAhw9WEWSpJnQ5ymmH1FVn8YxEJK0y+tzienUqdXdgFV0TMMtSdq19HmKafq9EI8D32Dy9jdJ0i6szz0IZ1WVpEVovleOvnee/aqq3jdAPZKkGTHfGcT3Otr2Bs4GDgAMCEnahc33ytEL55aTvBA4l8lLfK4ALtzWfpKkXcO89yCS7A+cB5wOrAWOqqqHF6IwSdK45rsH8dvAqcAa4BVV9d0Fq0qSNLr5Bsq9A3gx8B5gU5JH2+c7SR5dmPIkSWOZ7x7EDo+yliTtOgwBSVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVKnwQIiyceSbE7y1am2/ZPcmOTe9r1fa0+Si5NsSHJnkqOGqkuS1M+QZxCXAq/bqu18YF1VrQTWtXWAE4GV7bMa+OiAdUmSehgsIKrqs8C3tmo+mcmrS2nfp0y1X1YTtwL7Jlk6VG2SpO1b6HsQB1fVgwDt+6DWfgjwwFS/ja3tKZKsTrI+yfotW7YMWqwkLWazcpM6HW3V1bGq1lTVqqpatWTJkoHLkqTFa6ED4qG5S0fte3Nr3wgcOtVvGbBpgWuTJE1Z6IC4FjizLZ8JXDPVfkZ7mukY4JG5S1GSpHHsMdSBk1wOvBY4MMlG4ALg/cCVSc4G7gdOa91vAF4PbAC+D5w1VF2SpH4GC4iqess2Np3Q0beAc4aqRZK042blJrUkacYYEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSeo0UwGR5HVJvpZkQ5Lzx65HkhazmQmIJLsDvwecCBwOvCXJ4eNWJUmL18wEBHA0sKGq7quqvweuAE4euSZJWrRmKSAOAR6YWt/Y2iRJI9hj7AKmpKOtntIpWQ2sbqvfTfK1QataXA4Evjl2EbMgv3Pm2CXoR/lvc84FXf+p3GEv6dNplgJiI3Do1PoyYNPWnapqDbBmoYpaTJKsr6pVY9chbc1/m+OYpUtMXwJWJjksyV7Am4FrR65JkhatmTmDqKrHk7wd+FNgd+BjVXX3yGVJ0qI1MwEBUFU3ADeMXcci5qU7zSr/bY4gVU+5DyxJ0kzdg5AkzRADQk5xopmV5GNJNif56ti1LEYGxCLnFCeacZcCrxu7iMXKgJBTnGhmVdVngW+NXcdiZUDIKU4kdTIg1GuKE0mLjwGhXlOcSFp8DAg5xYmkTgbEIldVjwNzU5zcA1zpFCeaFUkuBz4P/FiSjUnOHrumxcSR1JKkTp5BSJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQWvSSLEtyTZJ7k3w9ye+2MSHz7fPuhapPGosBoUUtSYBPAZ+uqpXAy4EXAL+xnV0NCO3yDAgtdscDj1XV/wCoqh8Cvwb8myRvS/KRuY5Jrk/y2iTvB56X5CtJPtG2nZHkziR3JPl4a3tJknWtfV2S5a390iQfTXJTkvuSvKa99+CeJJdO/d7PJfl8ktuTfDLJCxbsfxUJA0L6SeC26YaqehS4n228s72qzgf+rqqOrKrTk/wk8B+B46vqCODc1vUjwGVV9U+ATwAXTx1mPybh9GvAdcBFrZZXJDkyyYHAe4CfqaqjgPXAec/EHyz11fl/AGkRCd2z126rvcvxwFVV9U2Aqpp7f8ErgVPb8seB35ra57qqqiR3AQ9V1V0ASe4GVjCZNPFw4HOTq2DsxWTKCWnBGBBa7O4G/tV0Q5J9mMxw+wg/epb93G0co2+YTPf5Qft+Ymp5bn0P4IfAjVX1lh7HlQbhJSYtduuA5yc5A/7xFawXMnnV5X3AkUl2S3Iok7fvzfmHJHtOHeNfJzmgHWP/1v6/mcyOC3A68Bc7UNetwLFJXtaO+fwkL9/RP07aGQaEFrWazFb5RuC0JPcC/wd4jMlTSp8D/i9wF/A7wO1Tu64B7kzyiTb77W8AtyS5A/hg6/MrwFlJ7gR+kSfvTfSpawvwS8Dlbf9bgR9/un+n9HQ4m6skqZNnEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOv1/El3en6cDX90AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target 分布，看看各类样本分布是否均衡\n",
    "sns.countplot(data.Outcome);\n",
    "pyplot.xlabel('Outcome');\n",
    "pyplot.ylabel('Number of diabetes');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "样本数量相差不是很多"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = data.drop([\"Outcome\"], axis=1)\n",
    "y_train = data['Outcome']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练 default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy of each fold is:  [0.75324675 0.74025974 0.78571429 0.79738562 0.77124183]\n",
      "cv acc is: 0.7695696460402341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()\n",
    "\n",
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "acc = cross_val_score(lr, X_train, y_train, cv=5, scoring='accuracy')\n",
    "print('accuracy of each fold is: ',acc)\n",
    "print('cv acc is:', acc.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid search for LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.1, 1, 10, 100, 1000, 10000, 100000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 正则化的 Logistic Regression及参数调优\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001,0.1,1, 10,100,1000,10000,100000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='accuracy')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00079994, 0.00099998, 0.00100002, 0.0006    , 0.00079994,\n",
       "        0.00079999, 0.00100007, 0.00079999, 0.00099993, 0.00119996,\n",
       "        0.00100002, 0.00100002, 0.00079999, 0.00080004, 0.00160007,\n",
       "        0.00079999]),\n",
       " 'std_fit_time': array([3.99971008e-04, 9.53674316e-08, 1.90734863e-07, 4.89901382e-04,\n",
       "        3.99971008e-04, 3.99994861e-04, 1.16800773e-07, 3.99994861e-04,\n",
       "        0.00000000e+00, 4.00066376e-04, 1.16800773e-07, 6.32485093e-04,\n",
       "        3.99994861e-04, 4.00018706e-04, 8.00096994e-04, 7.48353705e-04]),\n",
       " 'mean_score_time': array([0.00040002, 0.        , 0.00059996, 0.00019999, 0.00039997,\n",
       "        0.00019999, 0.00019999, 0.00019999, 0.00019999, 0.00020003,\n",
       "        0.00019999, 0.00019999, 0.00040002, 0.00020003, 0.00019999,\n",
       "        0.00040002]),\n",
       " 'std_score_time': array([0.00048992, 0.        , 0.00048986, 0.00039997, 0.00048986,\n",
       "        0.00039997, 0.00039997, 0.00039997, 0.00039997, 0.00040007,\n",
       "        0.00039997, 0.00039997, 0.00048992, 0.00040007, 0.00039997,\n",
       "        0.00048992]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.1, 0.1, 1, 1, 10, 10, 100, 100, 1000,\n",
       "                    1000, 10000, 10000, 100000, 100000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'},\n",
       "  {'C': 10000, 'penalty': 'l1'},\n",
       "  {'C': 10000, 'penalty': 'l2'},\n",
       "  {'C': 100000, 'penalty': 'l1'},\n",
       "  {'C': 100000, 'penalty': 'l2'}],\n",
       " 'split0_test_score': array([0.64935065, 0.74025974, 0.75974026, 0.75974026, 0.75324675,\n",
       "        0.75324675, 0.75324675, 0.75324675, 0.75324675, 0.75324675,\n",
       "        0.75324675, 0.75324675, 0.75324675, 0.75324675, 0.75324675,\n",
       "        0.75324675]),\n",
       " 'split1_test_score': array([0.64935065, 0.70779221, 0.74025974, 0.74675325, 0.74675325,\n",
       "        0.74025974, 0.74025974, 0.74025974, 0.74025974, 0.74025974,\n",
       "        0.74025974, 0.74025974, 0.74025974, 0.74025974, 0.74025974,\n",
       "        0.74025974]),\n",
       " 'split2_test_score': array([0.64935065, 0.75974026, 0.77272727, 0.78571429, 0.78571429,\n",
       "        0.78571429, 0.77272727, 0.77922078, 0.77922078, 0.77922078,\n",
       "        0.77922078, 0.77922078, 0.77922078, 0.77922078, 0.77922078,\n",
       "        0.77922078]),\n",
       " 'split3_test_score': array([0.65359477, 0.77777778, 0.79084967, 0.79738562, 0.79738562,\n",
       "        0.79738562, 0.79738562, 0.79738562, 0.79738562, 0.79738562,\n",
       "        0.79738562, 0.79738562, 0.79738562, 0.79738562, 0.79738562,\n",
       "        0.79738562]),\n",
       " 'split4_test_score': array([0.65359477, 0.75816993, 0.75816993, 0.76470588, 0.77777778,\n",
       "        0.77124183, 0.77124183, 0.77124183, 0.77124183, 0.77124183,\n",
       "        0.77124183, 0.77124183, 0.77124183, 0.77124183, 0.77124183,\n",
       "        0.77124183]),\n",
       " 'mean_test_score': array([0.65104167, 0.74869792, 0.76432292, 0.77083333, 0.77213542,\n",
       "        0.76953125, 0.76692708, 0.76822917, 0.76822917, 0.76822917,\n",
       "        0.76822917, 0.76822917, 0.76822917, 0.76822917, 0.76822917,\n",
       "        0.76822917]),\n",
       " 'std_test_score': array([0.00207782, 0.0236769 , 0.0167991 , 0.0182561 , 0.01926293,\n",
       "        0.02080024, 0.0193753 , 0.01993154, 0.01993154, 0.01993154,\n",
       "        0.01993154, 0.01993154, 0.01993154, 0.01993154, 0.01993154,\n",
       "        0.01993154]),\n",
       " 'rank_test_score': array([16, 15, 14,  2,  1,  3, 13,  4,  4,  4,  4,  4,  4,  4,  4,  4]),\n",
       " 'split0_train_score': array([0.6514658 , 0.752443  , 0.76710098, 0.7752443 , 0.77198697,\n",
       "        0.77687296, 0.77687296, 0.77687296, 0.77687296, 0.77687296,\n",
       "        0.77687296, 0.77687296, 0.77687296, 0.77687296, 0.77687296,\n",
       "        0.77687296]),\n",
       " 'split1_train_score': array([0.6514658 , 0.76547231, 0.77850163, 0.78175896, 0.78013029,\n",
       "        0.78013029, 0.77850163, 0.77850163, 0.77850163, 0.77850163,\n",
       "        0.77850163, 0.77850163, 0.77850163, 0.77850163, 0.77850163,\n",
       "        0.77850163]),\n",
       " 'split2_train_score': array([0.6514658 , 0.74104235, 0.77361564, 0.77687296, 0.78013029,\n",
       "        0.78175896, 0.78013029, 0.78013029, 0.78013029, 0.78013029,\n",
       "        0.78013029, 0.78013029, 0.78013029, 0.78013029, 0.78013029,\n",
       "        0.78013029]),\n",
       " 'split3_train_score': array([0.6504065 , 0.74796748, 0.76260163, 0.76747967, 0.76747967,\n",
       "        0.76910569, 0.77073171, 0.77073171, 0.77073171, 0.77073171,\n",
       "        0.77073171, 0.77073171, 0.77073171, 0.77073171, 0.77073171,\n",
       "        0.77073171]),\n",
       " 'split4_train_score': array([0.6504065 , 0.74634146, 0.77560976, 0.78536585, 0.78861789,\n",
       "        0.78699187, 0.78699187, 0.78536585, 0.78373984, 0.78373984,\n",
       "        0.78373984, 0.78373984, 0.78373984, 0.78373984, 0.78373984,\n",
       "        0.78373984]),\n",
       " 'mean_train_score': array([0.65104208, 0.75065332, 0.77148592, 0.77734435, 0.77766902,\n",
       "        0.77897196, 0.77864569, 0.77832049, 0.77799529, 0.77799529,\n",
       "        0.77799529, 0.77799529, 0.77799529, 0.77799529, 0.77799529,\n",
       "        0.77799529]),\n",
       " 'std_train_score': array([0.00051895, 0.00825811, 0.00581241, 0.00609553, 0.00732259,\n",
       "        0.00591907, 0.00524704, 0.00474616, 0.00428564, 0.00428564,\n",
       "        0.00428564, 0.00428564, 0.00428564, 0.00428564, 0.00428564,\n",
       "        0.00428564])}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7721354166666666\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "c在中间，L1正则\n",
    "l1, c=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOXZ//HPlcm+AIEEhIR9kx0ksiqbG1WrooK4i1XcsNa2Pm1tH+vS9qftY60KgohrXVBQlCoFUUSUfRFkkxCQJSwGQkISss7M9ftjJiGEhExCJpPler9e85o5Z+4z802Ac3Gf5b5FVTHGGGPOJCjQAYwxxtR9ViyMMcZUyoqFMcaYSlmxMMYYUykrFsYYYyplxcIYY0ylrFgYY4yplBULY4wxlbJiYYwxplLBgQ5QU+Li4rRDhw6BjmGMMfXK+vXrj6pqfGXtGkyx6NChA+vWrQt0DGOMqVdEZK8v7ewwlDHGmEpZsTDGGFMpKxbGGGMqZcXCGGNMpaxYGGOMqZQVC2OMMZWyYmGMMaZSViyMMcZUyoqFaTBueHklN7y8MtAxjGmQrFgYY4ypVIMZ7sOYPaH/5331YUBzGNMQWc/CVNmkhZOYtHBSoGMYY2qRFQtjjDGVsmJhjJ/V1Z7Y4NevY/Dr1wU6xmksV9XUVi4rFsYYYyplJ7iN8bNth7ICHcGYs2Y9C2OMMZWynoVpMP78znbPi7p3esCYes96FsYYYyplPQtTZRNf3Op5MTawOeqL9kW7Ah3BmLPm156FiIwVkR0ikiIivy/n/edEZKP3kSwimaXe+7uIbBWR7SLygoiIP7Ma4y93fFDIHR8UBjqGMWfFbz0LEXEA04BLgFRgrYjMV9VtxW1U9eFS7R8EBnhfDwOGA329b38LjASW+iuvMY1NXT3HY7mqprZy+bNnMQhIUdXdqloIzAauPkP7G4H3vK8VCAdCgTAgBPjJj1mNMcacgT+LRQKwv9RyqnfdaUSkPdARWAKgqiuBr4BD3sciVd3ux6ymCnILXeQWugIdwxhTi/x5gru8cwxaQduJwFxVdQGISBegB5DofX+xiIxQ1WWnfIHIZGAyQLt27WoktKk/1OnEnZODKycHd3Y24QWev165332HhIQiISGeR2hIqdel1gfZxYDG+MqfxSIVaFtqORE4WEHbicADpZbHAatUNQdARP4LDAFOKRaqOhOYCZCUlFRRITJ1kLrduE+cwJ2VVbKzd2Vne3b+WVm4s3Nw52Tjyi71XnY2rpxsz3vZ2bhzc0/5zNbe57033uRbCIfjZOEICUFCgpHg4keQZ9kR5HntEM8jSBAHSJAiQSBBbkTUsywuRNzeZyfgRHAyIO44AFmPXACOYHB4voPgEMThfQ4O8T6HlloOPfkcElbmORRCwrzLYUhoOISEIiHhEBKOhIZ5nyPAEQKOUM8jyFFzf4imUfFnsVgLdBWRjsABPAXhtH/FItIdiAVKT3G2D7hbRP4fnh7KSOBffsxqqkKVIIWiw4e9O/Ic3NlZnucc744969Qduysn55TC4D5xAvTM9V1CQwmKicERHe15bhJDcMuWBMVE44iOKVkXFBVFkDuTfW/9heAQN4mjL0cL8tHCfLSgAC0qQAsLTz6KCtEiJ1pUhDpPoE4X6nSiLgW3oG7BnQ+aK95lcHuf1ft+yWtX6XUVXbDXBIDMjek1/AfhI1FESj+DBAmdcSNA8oBzA5OrAl3cbsBy+aqL201QjP8PC/utWKiqU0SmAIsAB/Caqm4VkSeBdao639v0RmC26il7jrnAGGAznkNXC1X1P/7Kanzjyskh4+136HIwj2A3pIwaXXHj4GDPTr5Jk5KdfUj7doRHxxDUJObkzj4mmqBo704/Joag6GgcMd7XYWHlf3ZeJhxYB/vXwP7FsGMdFGbTe6D3/WPvQ0gEBIdDhPc5JByCIyCkCYREetdFVPx8pveCwz2fUfyZjhAQQVXB5fIUoVKPryaORhRGvPof1OUGt6vUswvc7vKfnU60KB+KCsFZiDoLoagQdRZ5l4vAWYQWFYKr6OSyq8izrbMIXE7U5fQsu1wnl10usnf9AEB0+841+xflLOXs9dyXYrl8k7N3F0W1cMecX79CVRcAC8qse6zM8uPlbOcC7vFnNuM7V2Ymx976N8fefht3VhaF4ZAVBj1/84Rnxx7t3ek3aVKys5fwcGrk1hhVSN8F+1d7HqlrIW07oCBB0KoX9LsBEgex9I//S16Bg58tDsy1ECIC3sNYRESUrHcGe34PYV26BCRXRTZc3BOAy99YHOAkp1pguapkwcU9QT0nef3J7uA2FXKmp3PsjTfIeOdd3Lm5xFxyMS3uuZevfjUegNgJE2r+Swtz4eAGb3FY43nkHfO8F94UEgdBr2uh7SBIOA/CYko2zc1/oubzGGMAKxamHEU/pXHstdfIeP99tKCAJj8bS4t77iW8e7ea/SJVOJ5aqjCshp+2gNvpeT+uG5x7ObQd7Hm06Ap2BZMxAWHFwpQoOnCA9FdfJXPuh6jLRdMrr6TFPfcQ1qljzXyBsxAOf3/ykNL+NZB9yPNeSCQkDIThv/IUhsQkiGxeM99rjDlrViwMhfv2cXTmTI5//AmI0Oyaa2gx+W5C27atfOMzyUk72WPYvwYOfgeuAs97zdpDhwu8vYZB0LKX57JSY0ydZP86G7GC3bs5OmMGWZ9+hgQHE3vDDbT4xZ2EtGlT9Q9zOSFt28mT0PtXQ8Yez3uOUGgzAAbdfbI4xJxToz9LXfbEzZ5Tj5cHOEdZlqtqGnsuKxaNUP6OHRydMYPshYuQ8HCa3347zSfdQUjLlj5/RrDDDTsXn+w1HFgPhTmeN6NbeYrC+Xd5nlv3g+AKLoM9S6rK7qMnWLrjCC+M+AUOKWTP0hR6tWlKz9ZNiI/xz/ca09hYsQAmLfQM1/j62NcDnMS/8jZv4eiMGeR8+SVBUVG0uPtumt9xO8HNq3BuwFXE+b2OEh9bAO9cD+KAc3pD/5s8Vyq1HQTN2oEfR5TPLXSyclc6S3ccYWlyGvuP5QEQFhKHahB/X7ijpG3LmDB6tWlCzzZN6NWmKb3aNKFtbCRBQTbivTFVYcWiEcjd8B1Hp0/nxDffENSkCXFTptD81ltwNG1apc85kHOAD7/9C/P7tuB4UBBNw5sTGd6UqJAYIh3HiDyyjIjMtUQGRxIZEklUSJTntXc5MjiSiJCIk+tDTr4XGhRa4X0ZqsquIydYuiONr5OPsHr3MQpdbiJCHAzv0oLJIzozqls8v5k7DIBXblnLtkNZbD14nG0Hs9h2KItlO4/icnvu+4wOC6Zna08B8RSRJnRtGUNosF1pZUxFrFg0UKpK7uo1HJ0+ndzVq3HExhL/618Te9ONOKKjff4cp9vJstRlfJD8ASsOrEBUSSpykhISztB2o8gtyiXXmUtuUS6Hcw+fspzrzK38C7yCJZiIkIiS4hHuiKCoKITsvCCOZQsn8oPBHUpsRDTn929Oz1bx9DgnnqZhWUSEFJHuzODHkEQgCHdQDud1iGZIp+YlBSi/yEXyT9lsO5jFVm8B+WDd/pLRc0McQteWMaf0Qnq0jiEmPKRKv3djGiorFg2MqnLi2285On0GeRs24IiPo+XvfkfsDRMIioz0+XMOnzjMRzs/4sOdH5KWm0bLiJbc0+4yrv32VT6JjiU7yMFTw58642e41U2+M/+U4lH2+UTRCfKceZwoOsGhrOP8eCyDAxmZHMvNQYOO43AUERHhpHl0ES7JJ9eZx/e5yvc/Aj+W+UJvx2DE+yMAcIijpDcTGezt6YREEhkTSefYSHr1jcDpDCEnz0FGThBHs5TF+93MS3ag7lDUHUbrmCZ0bxlHz3Pi6dOmJf0T4mjVNAJjGhsrFg2EqpKzZAlHp88gf8sWglu3ptX//olm119f8RhLZbjcLpYfXM6cHXNYdmAZqsqwhGE8OvhRRiaOJPjdGyC8Bd9E+razDJIgz845JBLK2eREgZPlKUdZmnKEr3cc4UCm59xD15bR3Nw9nlHdW5LUIZaw4JMjpaoq+a58T5EpyvMUHm/x+eXnTwBufjPkzpJ1ec68ktcnik6Q68wlLTfNs75UwVLUM4JZi1OjZgFrXbD2AHAAdI0gGkaIhBMeHEGT0GhiI6KJjYghKuTUQ2vFz06OA8Jnuz/z6fdWk1SVAqdS6HRR4HRT4HRRUOSmwOkmLy8MRXjyq7drPdeZ5OaFA1guH+XmhVMbp+CsWNRz6nKRvXgxR6fPoGDHDkLatuWcp56k2dVXI6GhPn3GkdwjJb2IQycO0SK8BXf2vpPrul5HYox3SpGftkLKFzDmTxT98E71sqqyMy2HpTvSWLrjCGv3HKPIpUSFOhjWJY77R3dmZLd4EmMr7gGJCBHBEUQER5xWgILxDP1xUw8fhygvlSvflX9Kr6e4t1O8fCwvm73HMkg9nsmh7OMcPZFNRnYOGVLI3qBMHI4jhIYW4XAUolJAoTsfxTNKaXGP5/ffnDYNfWBFeZ7m7Ps+sDnKslxVEwXOfP9fim7Fop5Sp5OsBQs4OuNlCnfvJrRjR9o88zRNrrjCM5BdJdzqZtWhVczZMYel+5fiVCeDWw/mN0m/YUzbMYQ4yhyrXzHVc5d10i+gCsUip7j3sOMIy5JP9h66tYpm0vCOjOoWT1KH5gE9uVy6ALWghc/bFThd7Pwpp+QkevEJ9ROFLkAJdrjo2DKY/fnrkaAiRrYdTn6Ri9wiF3mFLnILneQXuckrcpJX6Cbf6fsw06FBQYSHOogIcRAREkR4aDBRoUGEhwQT6V2OCA0iIiSYyFAHESVtPa/DQxz86eunQZTnL/nfavzW/OehxZ7Dm5bLNw8tfgoR/0/nY8WintHCQo7Pn8/Rma9QtG8fYd26kfDcP4m59FLEUfnENul56Xyc8jFzk+eSmpNKs7Bm3NLzFq7vdj3tm7Qvf6Osg7B5DiTdWekQHKpK8k8new/r9p7sPQzvEscDo7swsns8Cc3q/3H/sGAHvROa0jvh5FVlbrey71iu9yT6cbYezGJXWg+UIDa7QokKcxAZGkyTMAetI4OJCvPszEueQz3rittFhTqIDDv5HB0aTESoo0aK6xMrPXfTX9ix51l/Vk0KCSkELJevinP5mxWLesJdUMDxjz7i6Cuv4Dx4iPBevWg19UWix4ypdHpQVWXt4bXMSZ7DF/u+wOl2MrDVQB4c8CAXt7+YUEclh6tWzwB1wdD7y307O7+I5SnpfJ2cxtc7jnDweD4A3VvFcOfwjozsHk9S+8D2HmpLUJDQIS6KDnFRXNHXM3ff4NefAWD5pA8DGc2Ys2LFoo5z5+WR+cEHpL/6Gs60NCL696f1448TdeGFlc4XkZmfySe7PmFu8lz2ZO0hJjSGid0nMr7beDo16+RbgPwsWPc69LwaYjsAsCe4M87CWGZ8vYulO9JYtycDp1uJDgtmeJcWPHhRV0Z2i6dNA+g9GGM8rFjUUa6cE2S89y7HXn8D17FjRA4aRJu/P0Pk4MFnLBKqyndp3zEneQ6f7/mcQnch/eL78dcL/sql7S8lPDi8akE2vAUFWTDsl4DnOP2xvVfhLIjj6T0/cO45Mfziwo6M6taSge1jG0XvwZjGyIpFHePKyuLYv//Nsbf+jfv4caIuuIC4++4lcuDAM26XVZjFf3b9h7nJc0nJTCE6JJpru17L+O7j6RZbzXkoXEWwajq0v8Az0RAwd30qzoI4ouPWsvju39Ha7jkwplGwYlFHODMyOPbGm2S88w7unByix4wh7t57iOjbt8JtVJXNRzfzwY4PWLRnEfmufHq16MUTw55gbIexnvsbzsbWeZCVClf+E4Ail5vpS3cREp5GZPPNViiMaUT8WixEZCzwPJ7bnWap6tNl3n8OGO1djARaqmoz73vtgFlAW0CBy1V1jz/zBoLzyBHSX3+DjNmz0bw8Yi69lLh77yG8R8Uz6uYU5vDZ7s+YkzyHHRk7iAiO4MrOVzK+23h6tqihKzVUYfkLENcdulwCwCcbD5KakUezhI3+HCfQGFMH+a1YiIgDmAZcAqQCa0VkvqpuK26jqg+Xav8gMKDUR7wF/FVVF4tINBTf4dQwFB0+TPqsV8mcMwctKqLJFVcQd89kwrp0qXCbrelbmbNjDgt+XECeM49zm5/L/w75Xy7veDnRob6P9+ST3V/BT5vhqqkQFITLrbz0VQo9WzfhaFRqzX6XMabO82fPYhCQoqq7AURkNnA1sK2C9jcCf/a27QkEq+piAFXN8WPOWlWYmkr6zFfInDcPVGl69VXETZ5MaPvy73HILcrlvz/+lznJc9iavpVwRzhjO45lQrcJ9I7rXekVUdW24kXPvBR9JwCwYPMhdh89wUs3n8cT6/zzlcaYusufxSIB2F9qORUYXF5DEWkPdASWeFd1AzJF5CPv+i+A36uq77e41jEFP/5I+sxXOD5/PhIURLPrr6PFL+4iNDGh3PY7ju1gTvIcPtv9GTlFOXRp1oU/DPoDV3a+kiahTfwb9vBm2LUELnoMgsNwu5WpS1Lo0jKasb3OsWJhTCPkz2JR3n95K7onfSIwt1QxCAYuxHNYah/wPnAH8OopXyAyGZgM0K5du7NP7Af5ycmkvzyTrP/+FwkNpfktN9P8zjsJadXq9LbOfBbtWcSc5DlsOrKJ0KBQLu1wKRO6T6B/fH//9SLKWjEVQqI8d2wDX2z/iR0/ZfPcDf1s0iBjGil/FotUPCeniyUCBytoOxF4oMy235U6hPUxMIQyxUJVZwIzAZKSkqo9OMrEF7d6Xoyt7iecLn/bNo5On0H24sUERUbS4s5JNL/jDoLj4k5ruztzN3OS5/DJrk/ILsymQ5MOPJL0CFd1vopm4c1qLpQvjqfClrlw/t0QEYuqMvWrFNo1j+TnfasxN7cxpkHwZ7FYC3QVkY7AATwF4bThQEWkOxALrCyzbayIxKvqEWAMUC8OfuRt2sTRl6aT8/XXBMXEEHf/fcTeeivBsbGntCt0FbJ472LmJM9h/U/rCQ4K5pJ2lzC++3iSWiXVXi+irNUzPFdCDbkPgGU7j/J96nGevrYPwQ674c6YxspvxUJVnSIyBViE59LZ11R1q4g8CaxT1fnepjcCs1VVS23rEpHfAl+KZ6+5HnjFX1lrQu7atZ6pS1esxNGsGfG/eojYm2/GERNzSru9WXuZmzyXj1M+JrMgk7YxbXl44MNc3flqWkT4PuKpX+Qfh3VvQK9rINZzwn3qkp20bhrOteclBjabMSag/HqfhaouABaUWfdYmeXHK9h2MVDxHWl1gKqSu3IlR1+aTu66dTji4mj5yCPETryBoKioknZFriK+3P8lc3fMZfXh1QRLMKPbjeb6btczpPUQgqSO/I99/ZtQmA3DHgRg9e501u7J4ImretkwHsY0cnYHdzWoKjlff83R6dPJ3/Q9wa1a0erRR2k2YTxB4SfHXtqfvZ8Pkz9kXso8juUfo01UGx4c8CDjuowjPjI+gD9BOZyFnqE9OlwIbTy3u0z9KoW46DBuOL9tJRsbYxo6KxZVoG432V98wdEZMyjYtp2QhATOefxxml47jiDvrHROt5Ov93/NnOQ5rDi4AhFhROIIJnSbwLA2w3AEVT7nREBs/QiyD8JVLwDw3b4Mvtl5lEcvP5fwkDqa2RhTa6xY+EBdLrL+u5D0l2dQsDOF0Pbtaf23v9H051ciIZ4Z5Q7lHOLDnR8yb+c80vLSaBnZkvv63ce4ruM4J8r/Ux6eleKhPeJ7QJeLAZj2VQrNIkO4eXAFEyIZYxoVKxZnoEVFHP/Pp6S//DKFe/cS2qUzbf7v/2jys7GIw4HL7eJbby/imwPfoKoMTxjOn7r9iQsTLyQ4qJ78enctgbStcPVLIMLWg8f5Ynsav76kG1Fh9eRnMMb4le0JyuEuLOT4R/NIf+UVig4cIKxHDxKef56YSy5GgoJIy03jo50f8eHODzl84jBxEXH8ovcvuK7bdSREl39Hdp224gWIPgf6XA/AS1/tIiYsmNuHdQhsLmNMnWHFohR3fj6Zc+aS/uqrOA8fJrxvX1r96Y9EjxqFoqw4uJI5yXNYun8pLnUxtPVQ/uf8/2FU21GEBIUEOn71HNoEu5fCxY9DcBgpadks2HKI+0d1pmlE+T9Th8Lf1mZCn/Vs7edhUIxpxKxYAOJWorOKSLn4ElxHjxKRNJDWf/0LUcOGkZ6fzvtbXmVu8lwO5BygeXhzbut1G9d3vZ52TermECNVsmIqhEbDwEmAp1cRHuzgzuEdAxzMGFOXNPpiUbh3L232nsDhVsKHnUfcc/8kIimJNYfX8MHXv2XJ/iU43U7OP+d8HjrvIS5qdxGhjtBAx64Zmfthy4cw+F6IaMbe9BN8sukgk4Z1oEV0WKDTGWPqkEZfLELatiU3OpgTMSH0f+lZ5uyaz5yPn2Bv1l6ahjXlxnNv5Ppu19OpaadAR615q2d4nr1De8z4eheOIOHuEQ3wZzXGnJVGXywQYV2PEFZ1drFlzkUUuYsY0HIA9/S9h0s7XEqYo4H+DzsvE9a/Ab2vhWZtOXQ8j7nrU5l4fjtaNQmvdHNjTOPS6ItFanYq08cUEl4I47vdwPXdrqdrbNdAx/K/9W9AYU7J0B4vf70bVbhnZP3tVbw+9vVARzCmwWr0xaJtk7bc+U0oXdKC+Pndfwh0nNrhLPQcguo4Elr340h2Ae+t2ce15yWQGBsZ6HQNTl29emz1pA8DHaFclqtqaitXoy8WAD0PNbLhLLbMhexDcPVUAGZ9u5sil5v7RlU8/3dp798z1J/pGhz7fZmGwIYSbWxUPfNrt+wFnS8i40Qhb6/cy8/7taFjXFTl2xtjGiUrFo1NypeQts1zrkKE11fs4UShi/t97FUYYxonKxaNzYrnIaYN9L6OrPwi3lj+I5f1akX3c2Iq39YY02hZsWhMDm6EH5fBkHshOJR/r9xLVr6TKaMbwdVfxpizYsWiMVnxIoTGwMA7yC108uq3PzKqezx9EpsGOpkxpo6zYtFYZO6DrfNg4O0Q3pR3V+/j2IlCHhxj5yqMMZXza7EQkbEiskNEUkTk9+W8/5yIbPQ+kkUks8z7TUTkgIhM9WfO2Q/2YvaDvfz5FYG3ajqIwJD7yC9y8co3uxnaqQUD2zcPdDJjTD3gt/ssRMQBTAMuAVKBtSIyX1W3FbdR1YdLtX8QGFDmY54CvvZXxkYjLwPWvwm9r4OmicxdtZefsgp4bkL/QCczxtQT/uxZDAJSVHW3qhYCs4Grz9D+RuC94gURGQi0Aj73Y8bGYd3rUHQChj1IkcvN9KW7OK9dM4Z2bhHoZMaYesKfxSIB2F9qOdW77jQi0h7oCCzxLgcBzwKP+DFf4+As8Azt0Wk0nNOHj787wIHMPB4c0xURCXQ6Y0w94c9iUd6eSCtoOxGYq6ou7/L9wAJV3V9Be88XiEwWkXUisu7IkSNnEbUB2zwHcn6C4b/E5VZeWrqLXm2aMKp7fKCTGWPqEX+ODZUKtC21nAgcrKDtROCBUstDgQtF5H4gGggVkRxVPeUkuarOBGYCJCUlVVSIGi+323O5bKs+0Gk0n31/iB+PnmD6zedZr8IYUyX+LBZrga4i0hE4gKcg3FS2kYh0B2KBlcXrVPXmUu/fASSVLRTGBylfwJEfYNxM3ArTlqTQpWU0l/U6J9DJjDH1jN8OQ6mqE5gCLAK2Ax+o6lYReVJErirV9EZgtqpaz6CmrXgBmiRA72tZvP0ndvyUzZTRXQgKsl6FMaZq/DpEuaouABaUWfdYmeXHK/mMN4A3ajhaw3dgA+z5Bi79CxoUzNQlKbRvEcmVfVsHOpkxph6yO7gbqhUvQlgTOO92vk4+wuYDx7l/VGeCHfZHboypOttzNEQZe2HbxzDwDjQshqlLUmjTNJxxAxIDncwYU0/5VCxE5EMRucJ7/4Op61a9BBIEg+9l9Y/HWLc3g3tHdSY02P74jDHV4+veYzqeK5l2isjTInKuHzOZs5F7DDb8G/qMh6YJTF2SQnxMGBOS2la+rTHGVMCnYqGqX3gvZz0P2AMsFpEVIjJJREL8GdBU0brXSob22LAvg29TjjL5wk6EhzSyecaNMTXK5+MSItICuAO4C/gOeB5P8Vjsl2Sm6oryYfXL0PkiaNWLaUtSiI0M4abB7QKdzBhTz/l06ayIfAScC/wb+LmqHvK+9b6IrPNXOFNFmz+AE2kw/JdsOXCcL39I47eXdiMqzK9XSBtjGgFf9yJTVXVJeW+oalIN5jHVVTy0xzl9oONIXnp3AzFhwdw6tEOgkxljGgBfD0P1EJFmxQsiEusdt8nUFTs/h6PJMOwhdqbl8N8th7l9WAeaRtgpJWPM2fO1WNytqiWz2KlqBnC3fyKZalnxAjRJhF7X8NLSXYQHO7jzgo6BTmWMaSB8LRZBUmqYUu8seKH+iWSqLHU97F0OQ+9nb2Yhn2w8wC1D2tE8yv6IjDE1w9dzFouAD0RkBp45Ke4FFvotlamaFS9AWFM47zamf7qLYEcQd1/YKdCpjDENiK/F4nfAPcB9eCY1+hyY5a9QpgqO/Qjb58OwX3IwL5gPN6Ry46B2tGwSHuhkxpgGxKdioapuPHdxT/dvHFNlq14CccDge5m5dDeqcM/IzoFOZYxpYHy9z6Ir8P+AnkDJf1lV1Y51BFLuMfjubeg7gTSJ5b01G7nuvEQSmkUEOpkxpoHx9TDU68CfgeeA0cAkyp9ju156fezrgY5QPWtfhaJcGDqFV7/5kSKXm/tGWa/CGFPzfL0aKkJVvwREVfd6Jywa479YplJF+bDmZehyCRnRXfj3qr1c1a8NHeKiAp3MGNMA+dqzyPcOT75TRKbgmVO7pf9imUp9PxtOHIHhv+T15T+SW+jigdFdAp3KGNNA+dqz+BUQCfwSGAjcAtzur1CmEm43rJgKrfuRdc4QXl+xh7G9zqFrq5hAJzPGNFCVFgvvDXgTVDVHVVNVdZKqXqeqq3zYdqyI7BCRFBH5fTnvPyciG72PZBHJ9K7vLyIrRWSriHwvIjdU66drqJIXQvpOGPZL/r1qH9n5TqaMsV6FMcZ/Kj0MpaouERkoIqKq6usHe4vMNOC5PC0qAAAeBElEQVQSIBVYKyLzVXVbqc9+uFT7B4EB3sVc4DZV3SkibYD1IrKo9JAjjdqKF6BpO3K7XsmsecsY3T2e3glNA53KGNOA+XrO4jvgExGZA5woXqmqH51hm0FAiqruBhCR2cDVwLYK2t+I54orVDW51HccFJE0IB6wYrF/LexbCWOf5t21B8nILWLKmK6BTmWMaeB8LRbNgXROvQJKgTMViwRgf6nlVGBweQ1FpD3QEThtGHQRGYRnHKpd5bw3GZgM0K5dI5ngZ8ULEN6U/D43MfNfaxnWuQUD28cGOpUxpoHz9Q7uSdX47PLuw6joMNZEYK6quk75AJHWeCZcut17F3nZXDOBmQBJSUk+HyKrt47thu3/gQseZs7mTNKyC/jXxP6BTmWMaQR8vYP7dcrZ0avqnWfYLBVoW2o5EThYQduJwANlvrMJ8BnwJ19OpjcKK6eBI4SipLuZMeMHBraPZWinFoFOZYxpBHw9DPVpqdfhwDgq3vEXWwt0FZGOeO7LmAjcVLaRiHQHYoGVpdaFAvOAt1R1jo8ZG7YT6fDdO9B3AvNSXBzIzOMv43pTauR4Y4zxG18PQ31YellE3gO+qGQbp/cGvkWAA3hNVbeKyJPAOlWd7216IzC7zJVWE4ARQAsRucO77g5V3ehL3gZp7Sxw5uEaMoWX3kqhd0ITRnWLD3QqY0wj4WvPoqyuQKVnlFV1AbCgzLrHyiw/Xs52bwNvVzNbw1OUB2tmQtfL+PRQE/ak72bGLQOtV2GMqTW+nrPI5tRzFofxzHFhasOm9yD3KO6hU5j2SQrdWkVzac9WgU5ljGlEfD0MZeNIBErx0B5tBvD5ia4k/7SB5yf2JyjIehXGmNrj09hQIjJORJqWWm4mItf4L5YpsWMBHNuFDn2QqUtT6NAikiv6tA50KmNMI+PrQIJ/VtXjxQveYTf+7J9I5hQrXoRm7fg6eChbDmRx/6guBDt8/WMzxpia4etep7x21T05bny1fw3sX4UOeYAXl+4hoVkE1wxICHQqY0wj5GuxWCci/xSRziLSSUSeA9b7M5gBlj8P4c1Y0+xy1u/N4N6RnQgNtl6FMab2+brneRAoBN4HPgDyKHPHtalh6bvgh8/g/Lt44duDxMeEMT6pbeXbGWOMH/h6NdQJ4LT5KIwfrZwKjhA2tZnA8sUp/OmKHoSHOAKdyhjTSPl6NdRiEWlWajlWRBb5L1Yjd+IobHwX+k3k+dVZxEaGcNPgRjKqrjGmTvL1MFRc6YmHVDUDm4Pbf9a8As58dna6nSU/pHHXhZ2IDLXrCYwxgeNrsXCLSMl/bUWkAxUPN27ORmEurH0Fuv2Mf24UYsKDuXVo+0CnMsY0cr7+d/WPwLci8rV3eQTeSYdMDdv0LuSms7/HL/jv+4d5cEwXmoSHBDqVMaaR86lnoaoLgSRgB54ron6D54ooU5PcLs+cFQkDefaHFkSGOpg0vGOgUxljjM8DCd4FPIRnAqONwBA880+MOdN2pop++AyO7SZt7MvM/+QQd13YieZRoYFOZYwxPp+zeAg4H9irqqOBAcARv6VqrFa8CM3a81xqd4IdQdx1ofUqjDF1g6/FIl9V8wFEJExVfwC6+y9WI7RvFaSuIbP/ZOZ+d4gbz29Ly5jwQKcyxhjA9xPcqd77LD4GFotIBpVPq2qqYsWLEBHL1GODgSNMHtk50ImMMaaEr3dwj/O+fFxEvgKaAgv9lqqxOZoCP3zGiSEP89a3R7nuvEQSmkUEOpUxxpSo8qh0qvq1qs5X1cLK2orIWBHZISIpInLacCEi8pyIbPQ+kkUks9R7t4vITu/j9qrmrFdWvgiOUGYVXITT5ea+UdarMMbULX67LVhEHMA04BIgFVgrIvNVdVtxG1V9uFT7B/GcOEdEmuOZLyMJz81/673bZvgrb8DkHIGN75Hf6wZeXp/D1f0TaN8iKtCpjDHmFP4c73oQkKKqu729kNnA1WdofyPwnvf1ZcBiVT3mLRCLgbF+zBo4a2aCq5B3HT8nr8jF/darMMbUQf4sFgnA/lLLqd51pxGR9kBHYElVt63XvEN7FHUdy3MblJ/1PoeurWy6c2NM3ePPYiHlrKtoPKmJwFxVdVVlWxGZLCLrRGTdkSP18LaPje9AXgbzI68ju8DJ/aO6BDqRMcaUy5/FIhUoPVtPIhVfbjuRk4egfN5WVWeqapKqJsXHx59l3FrmdsHKqbjaJPGX75sw5tyW9E5oGuhUxhhTLn8Wi7VAVxHpKCKheArC/LKNRKQ7EItn+JBii4BLvfNmxAKXetc1HNv/Axl7WNL8BjLynDww2noVxpi6y29XQ6mqU0Sm4NnJO4DXVHWriDwJrFPV4sJxIzBbVbXUtsdE5Ck8BQfgSVU95q+stU4VVryAO7Yjf/qhPcO7NGVg+9hApzLGmAr5dUYdVV0ALCiz7rEyy49XsO1rwGt+CxdI+1bCgfWs7fEoPx1y8q/RXQOdyBhjzsimXwuEFS+iEc35w+6+JLVvypBOzQOdyBhjzsif5yxMeY4kw44FbEucwO7jbqaM6YJIeRd/GWNM3WHForatnIoGh/PogaH0SWjKyG717CouY0yjZMWiNuWkwabZ7Em8mk3HQqxXYYypN6xY1KY1M1FXIU8cGU33VjFc0qNVoBMZY4xPrFjUlsITsHYWP7W5mKXpTXhgTBeCgqxXYYypH+xqqNry3duQl8H/BV9Kx7gorujTOtCJjDHGZ9azqA0uJ6ycxvG485h7JIH7RnXGYb0KY0w9Yj2L2rB9PmTuZUbTW0loFsG4AQ1vAF1jAqmoqIjU1FTy8/MDHaXOCg8PJzExkZCQkGptb8XC31RhxYvkxXTg5Z/O5YlrOhPisA6dMTUpNTWVmJgYOnToYFcYlkNVSU9PJzU1lY4dO1brM2yv5W97l8PBDbzr+DlxMRGMH5gY6ETGNDj5+fm0aNHCCkUFRIQWLVqcVc/LioW/rXiRorDm/P3wQCaP6ER4iCPQiYxpkKpaKG54eSU3vLyy8oYNxNkWUisW/pT2AyQv5LPwK4mKiuamwe0CncgY4yfR0dElr8eOHUuzZs248sory237wAMP0L9/f3r27ElERAT9+/enf//+zJ07t0rfuWHDBhYuXHhWuX1l5yz8aeVU3I4wnvhpGHdd1pHIUPt1G9MYPPLII+Tm5vLyyy+X+/60adMA2LNnD1deeSUbN26s1vds2LCBLVu2MHbs2Gpn9ZX1LPwl+zB8/z7fRF2GK7w5tw1tH+hExphactFFFxETE1OtbXfu3Mlll13GwIEDGTFiBMnJyQDMnj2b3r17069fP0aPHk1eXh5PPvkk77zzTrV6JVVl/9X1lzUzUVcRjx0ZxR1jOhITXr3L1YwxVfPEf7ay7WBWpe22HfK08eW8Rc82Tfjzz3uddTZfTJ48mVmzZtG5c2eWL1/OlClT+Pzzz3niiSdYunQprVq1IjMzk4iICB577DG2bNnCv/71L7/nsmLhDwU5sPZVNsVcyFF3ApOGdQh0ImNMPZCZmcmqVau47rrrStY5nU4Ahg8fzm233cb48eO59tpraz2bFQt/+O5tyM/kyayLuOWC9sRGhQY6kTGNhq89gOIexfv3DPVnnCpRVeLi4so9h/HKK6+wevVqPv30U/r168f3339fq9nsnEVNczlh1TR+jOzL1qDu/OLC6t0AY4xpfGJjY2ndujXz5s0DwO12s2nTJgB2797NkCFDeOqpp4iNjeXAgQPExMSQnZ1dK9n8WixEZKyI7BCRFBH5fQVtJojINhHZKiLvllr/d++67SLygtSXu222fQyZ+3jm+CXcOKgdLWPCA53IGFPLLrzwQsaPH8+XX35JYmIiixYt8nnb2bNnM2PGDPr160evXr349NNPAXj44Yfp06cPffr04eKLL6Z3796MGTOGTZs2MWDAgPp7gltEHMA04BIgFVgrIvNVdVupNl2BPwDDVTVDRFp61w8DhgN9vU2/BUYCS/2Vt0aowooXOBrWliVF57F0RKdAJzLG1JKcnJyS1998841P23To0IEtW7acsq5Tp07lFpf58+efti4+Pp5169ZVMWn1+POcxSAgRVV3A4jIbOBqYFupNncD01Q1A0BV07zrFQgHQgEBQoCf/Ji1Zuz5Bg5t4nnXXVw3sB1tmkUEOpExpgJ16VxFfeDPw1AJwP5Sy6nedaV1A7qJyHIRWSUiYwFUdSXwFXDI+1ikqtv9mLVmrHiRnOBY5rou5L6RXQKdxhhjaow/exblnWPQcr6/KzAKSAS+EZHeQBzQw7sOYLGIjFDVZad8gchkYDJAu3YBHkojbTvs/JzX3BP4Wb8OtGsRGdg8xhhTg/zZs0gF2pZaTgQOltPmE1UtUtUfgR14isc4YJWq5qhqDvBfYEjZL1DVmaqapKpJ8fHxfvkhfLZiKkVBYbxedBH3j+4c2CzGGFPD/Fks1gJdRaSjiIQCE4GyZ2g+BkYDiEgcnsNSu4F9wEgRCRaREDwnt+vuYaisQ+j37zPXNYphvbvRpWX1bvM3xpi6ym/FQlWdwBRgEZ4d/QequlVEnhSRq7zNFgHpIrINzzmKR1Q1HZgL7AI2A5uATar6H39lPWtrXkbdLqYXjuWB0Xauwph64fUrPA/jE7/eZ6GqC1S1m6p2VtW/etc9pqrzva9VVX+tqj1VtY+qzvaud6nqParaw/ver/2Z86wUZKNrX+VLBtG1ex96tmkS6ETGmACo7SHK582bxz/+8Y+zzu0rG+7jbG34N1KQxdSCy/nzGOtVGGNqbohyp9NJcHD5u+lx48bVTFgf2XAfZ8NVhK6cxgbpSUznIZzXLjbQiYwxdcDZDFF+wQUX8Mc//pERI0YwdepUPvnkEwYPHsyAAQO49NJLSUvz3I42a9YsfvWrXwFwyy238NBDDzFs2DA6depUMlxITbKexdnY9gmSlcq0wt8wxXoVxtQN//09HN5cebvD3oH4fDlvcU4f+NnTZ5erCrKysli2zHOnQEZGBldddRUiwowZM3j22Wd55plnTtsmLS2N5cuXs3nzZiZMmFDjPQ8rFtWlinv58+yTBLLbjmFwx+aBTmSMaSAmTpxY8nrfvn1MmDCBw4cPU1BQQLdu3crd5pprrkFE6Nu3LwcOHKjxTFYsquvHrwk6/D3Ti+7mgYu6n/Vk6MaYGuJrD6C4RzHpM/9lqaaoqKiS1w888ACPPvool19+OV988QVPP13+zxcWFlbyWrXs/c9nz4pFNbmXv0CGNCPlnMsZ0TUu0HGMMQ3U8ePHSUhIQFV58803A5bDTnBXx09bCdr1Ja8VXso9Y3par8IYc4qzGaK8rMcff5xx48YxcuRIWrVqVYMpq0b80V0JhKSkJK2toXp13r3kb/qIW5u8zge/upygICsWxgTS9u3b6dGjR9U2qsOHofylvN+TiKxX1aTKtrXDUFWVdRD9fi6znWO4/aIBViiMqa8aUZGoCXYYqop01QxUXSxuch2X92kd6DjGGFMrrGdRFflZuNa+ykLXIMaNGYbDehXGmEbCehZVoBveJLgoh3kR13HNgLLzOBljTMNlPQtfuYoo/HYa37l7MHrMZYQ4rM4aYxoP2+P5aus8wnIP8UHoOK4fmFh5e2NMnTZp4SQmLZwU6Bj1hhULX6iS+9U/2elOoNfI6wkPcQQ6kTGmjikeonzjxo0MHTqUXr160bdvX95///3T2tbEEOUAGzZsYOHChTWSvzJ2GMoXu5cSmbGddx338cjg9oFOY4ypwyIjI3nrrbfo2rUrBw8eZODAgVx22WU0a9aspI2vQ5RXZsOGDWzZsoWxY8fWSPYzsZ6FD7KX/JM0bUarC28jMtTqqzGmYt26daNr164AtGnThpYtW3LkyBGft9+5cyeXXXYZAwcOZMSIESQnJwMwe/ZsevfuTb9+/Rg9ejR5eXk8+eSTvPPOO9XqlVSV7fkqc3gLMQeW8ZrcyJ3DugY6jTGmEs+seYYfjv1QabviNr6ctzi3+bn8btDvqpxlzZo1FBYW0rlzZ5+3mTx5MrNmzaJz584sX76cKVOm8Pnnn/PEE0+wdOlSWrVqRWZmJhERETz22GNs2bKFf/3rX1XOVlVWLCpxfMk/CdYwQobcRUx4SKDjGGPqiUOHDnHrrbfy5ptvEhTk20GczMxMVq1axXXXXVeyzul0AjB8+HBuu+02xo8fz7XXXuuXzGfi12IhImOB5wEHMEtVTxtbV0QmAI8DCmxS1Zu869sBs4C23vcuV9U9/sx7muOpRCd/zLtcyk0j+9bqVxtjqsfXHkBxj+L1sa/XeIasrCyuuOIK/vKXvzBkyBCft1NV4uLiyj2H8corr7B69Wo+/fRT+vXrx/fff1+TkSvlt3MWIuIApgE/A3oCN4pIzzJtugJ/AIarai/gV6Xefgv4h6r2AAYBaf7KWpHMpS+iquQMmEyzyNDa/npjTD1UWFjIuHHjSnoBVREbG0vr1q1LpkV1u91s2rQJgN27dzNkyBCeeuopYmNjOXDgADExMWRnZ9f4z1Aef57gHgSkqOpuVS0EZgNXl2lzNzBNVTMAVDUNwFtUglV1sXd9jqrm+jHr6fKPE77pLRbqEK6/aHitfrUxpv764IMPWLZsGW+88UbJJbFVudpp9uzZzJgxg379+tGrVy8+/fRTAB5++GH69OlDnz59uPjii+nduzdjxoxh06ZNDBgwoF6f4E4A9pdaTgUGl2nTDUBEluM5VPW4qi70rs8UkY+AjsAXwO9V1eXHvKfI/HYWzdy5HOh5F/ExYZVvYIxp1HJycgC45ZZbuOWWW3zapkOHDmzZsuWUdZ06dSp3/ov58+efti4+Pp7amprBn8WivFH2yk6eEQx0BUYBicA3ItLbu/5CYACwD3gfuAN49ZQvEJkMTAZo165dzSV3FiKrp7PS3ZOrfnZ5zX2uMabO8Me5iobMn4ehUvGcnC6WCBwsp80nqlqkqj8CO/AUj1TgO+8hLCfwMXBe2S9Q1ZmqmqSqSfHx8TUW/Pi692ladIQfOk2iddOIGvtcY4ypr/xZLNYCXUWko4iEAhOBsv2oj4HRACISh+fw027vtrEiUlwBxgDb/Jj1JFXyv36OZHciF115c618pTHG1HV+KxbeHsEUYBGwHfhAVbeKyJMicpW32SIgXUS2AV8Bj6hquvfcxG+BL0VkM55DWq/4K2tpWVsX0SpvF+sTb6FdXFRtfKUxxtR5fr3PQlUXAAvKrHus1GsFfu19lN12MVDrNzccW/wsedqM838+uba/2hhj6iwbG6qU7B830OH4GlbGT6BL6xaBjmOM8aO9t97G3ltvC3SMesOKRSkHFvydHA2nx5W/DHQUY0w9U9tDlM+bN49//OMfNZa/MjY2lFfukb10ObKIJU3HcWmHtpVvYIwx5ajJIcqdTifBweXvpseNG1fz4c/AioVXyvy/01Mh4We/CXQUY0w91q1bt5LXpYcoL10szuSCCy5g5MiRfPPNN1x77bV07NiRv/3tbxQWFhIfH8/bb79Ny5YtmTVrVsmIs7fccgstWrRg7dq1HD58mGeffbbGi4kVC+D2aZ8zLW0ua6JGMqxHr0DHMcachcN/+xsF2ysfojz/B08bX85bhPU4l3MefbTKWaozRDl4BiJctmwZABkZGVx11VWICDNmzODZZ5/lmWeeOW2btLQ0li9fzubNm5kwYYIVC38YmvkfoiWfmNGnXZRljDHVUp0hyotNnDix5PW+ffuYMGEChw8fpqCg4JSeS2nXXHMNIkLfvn05cODAWWUvT6MvFoUF+Vxb+AkbtQv9zx8R6DjGmLPkaw+guEfR/t9v1XiG6g5RXiwq6uQ9Xg888ACPPvool19+OV988QVPP33aTA8AhIWdHMPOc1dCzWr0xSL98F4ytSlHHXGBjmKMaQDOZojy8hw/fpyEhARUlTfffLMGElZPoy8Wrdt356HWMwC4OMBZjDH1X/EQ5enp6bzxxhsAJcOVV8fjjz/OuHHjSExMZNCgQRw6dKgG0/pO/NFdCYSkpCSt7lC9N7y8EoD37xlak5GMMbVk+/bt9OjRo0rb+PMwVF1V3u9JRNaralJl2zb6ngVYkTCmMWpMRaIm2B3cxhhjKmXFwhhjTKWsWBhjGoSGcv7VX87292PFwhhT74WHh5Oenm4FowKqSnp6OuHh4dX+DDvBbYyp9xITE0lNTeXIkSOBjlJnhYeHk5iYWO3trVgYY+q9kJAQOnbsGOgYDZodhjLGGFMpKxbGGGMqZcXCGGNMpRrMcB8icgTYexYfEQccraE4NclyVY3lqhrLVTUNMVd7VY2vrFGDKRZnS0TW+TI+Sm2zXFVjuarGclVNY85lh6GMMcZUyoqFMcaYSlmxOGlmoANUwHJVjeWqGstVNY02l52zMMYYUynrWRhjjKmUFQsvEXlKRL4XkY0i8rmItAl0JgAR+YeI/ODNNk9EmgU6E4CIjBeRrSLiFpGAXx0iImNFZIeIpIjI7wOdp5iIvCYiaSKyJdBZiolIWxH5SkS2e/8MHwp0pmIiEi4ia0RkkzfbE4HOVExEHCLynYh8GugspYnIHhHZ7N13VW+6UB9YsTjpH6raV1X7A58CjwU6kNdioLeq9gWSgT8EOE+xLcC1wLJABxERBzAN+BnQE7hRRHoGNlWJN4CxgQ5RhhP4jar2AIYAD9Sh31cBMEZV+wH9gbEiMiTAmYo9BGwPdIgKjFbV/v68fNaKhZeqZpVajALqxMkcVf1cVZ3exVVA9YeNrEGqul1VdwQ6h9cgIEVVd6tqITAbuDrAmQBQ1WXAsUDnKE1VD6nqBu/rbDw7wITApvJQjxzvYoj3EfB/iyKSCFwBzAp0lkCxYlGKiPxVRPYDN1N3ehal3Qn8N9Ah6qAEYH+p5VTqyM6vrhORDsAAYHVgk5zkPdyzEUgDFqtqXcj2L+B/AHegg5RDgc9FZL2ITPbXlzSqYiEiX4jIlnIeVwOo6h9VtS3wDjClruTytvkjnsMH79SlXHWElLMu4P8bretEJBr4EPhVmZ51QKmqy3s4OBEYJCK9A5lHRK4E0lR1fSBznMFwVT0Pz2HYB0RkhD++pFHNZ6GqF/vY9F3gM+DPfoxTorJcInI7cCVwkdbitc5V+H0FWirQttRyInAwQFnqBREJwVMo3lHVjwKdpzyqmikiS/Gc8wnkBQLDgatE5HIgHGgiIm+r6i0BzFRCVQ96n9NEZB6ew7I1fi6xUfUszkREupZavAr4IVBZShORscDvgKtUNTfQeeqotUBXEekoIqHARGB+gDPVWSIiwKvAdlX9Z6DzlCYi8cVX/IlIBHAxAf63qKp/UNVEVe2A5+/WkrpSKEQkSkRiil8Dl+KnwmrF4qSnvYdYvsfzC68rlxNOBWKAxd5L42YEOhCAiIwTkVRgKPCZiCwKVBbvBQBTgEV4TtZ+oKpbA5WnNBF5D1gJdBeRVBH5RaAz4fmf8q3AGO/fqY3e/zXXBa2Br7z/DtfiOWdRpy5VrWNaAd+KyCZgDfCZqi70xxfZHdzGGGMqZT0LY4wxlbJiYYwxplJWLIwxxlTKioUxxphKWbEwxhhTKSsWxlSBiORU3uqM288VkU7e19Ei8rKI7PKOsLpMRAaLSKj3daO6adbUbVYsjKklItILcKjqbu+qWXgGGeyqqr2AO4A472CIXwI3BCSoMeWwYmFMNYjHP7w3cm4WkRu864NE5CVvT+FTEVkgItd7N7sZ+MTbrjMwGPiTqroBvKPmfuZt+7G3vTF1gnVzjamea/HMt9APiAPWisgyPHdHdwD6AC3x3FH+mneb4cB73te9gI2q6qrg87cA5/sluTHVYD0LY6rnAuA97wipPwFf49m5XwDMUVW3qh4Gviq1TWvgiC8f7i0ihcXj/hgTaFYsjKme8oZFP9N6gDw8o5YCbAX6iciZ/g2GAfnVyGZMjbNiYUz1LANu8E7UEw+MwDOQ27fAdd5zF62AUaW22Q50AVDVXcA64AnvKLCISNfiuUJEpAVwRFWLausHMuZMrFgYUz3zgO+BTcAS4H+8h50+xDO/xhbgZTwz0B33bvMZpxaPu4BzgBQR2Qy8wsl5OEYDC/z7IxjjOxt11pgaJiLRqprj7R2swTOT2WHv/AxfeZcrOrFd/BkfAX+oQ/Ocm0bOroYypuZ96p3AJxR4ytvjQFXzROTPeOYH31fRxt4JnD62QmHqEutZGGOMqZSdszDGGFMpKxbGGGMqZcXCGGNMpaxYGGOMqZQVC2OMMZWyYmGMMaZS/x/2Q115g4PZpgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuracy' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LRL1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.1, 1, 10, 100, 1000, 10000, 100000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l1', random_state=None, refit=True,\n",
       "           scoring='accuracy', solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 用LogisticRegressionCV实现正则化的 Logistic Regression\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='accuracy', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[0.64935065, 0.75974026, 0.75324675, 0.75324675, 0.75324675,\n",
       "         0.75324675, 0.75324675, 0.75324675],\n",
       "        [0.64935065, 0.74025974, 0.74675325, 0.74025974, 0.74025974,\n",
       "         0.74025974, 0.74025974, 0.74025974],\n",
       "        [0.64935065, 0.77272727, 0.78571429, 0.77272727, 0.77922078,\n",
       "         0.77922078, 0.77922078, 0.77922078],\n",
       "        [0.65359477, 0.79084967, 0.79738562, 0.79738562, 0.79738562,\n",
       "         0.79738562, 0.79738562, 0.79738562],\n",
       "        [0.65359477, 0.75816993, 0.77777778, 0.77124183, 0.77124183,\n",
       "         0.77124183, 0.77124183, 0.77124183]])}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.40949014,  1.14777835, -0.10817025, -0.01145812, -0.12116277,\n",
       "         0.65833545,  0.28656635,  0.14298924]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0,  5, 10,  9,  9, 10, 10, 11],\n",
       "        [ 0,  8,  9, 10,  9,  9,  9,  9],\n",
       "        [ 0,  8,  9,  9, 10,  9,  9,  9],\n",
       "        [ 0,  7,  9, 10,  9, 10, 10, 10],\n",
       "        [ 0, 10, 10, 10, 10,  9, 10,  9]]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.n_iter_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以从迭代次数观察模型的复杂度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt0lfWd7/H3F0K4XwIECIQAKqAoKBABa2vVakur1dZ2FITO2Jmps85op9OZdkbXzGpn7Jozs07XmZ5zVj3njNPT6SpBKFqljCCI1eq0Q4BwUW6ClLqTcI2QCOEScvmeP/azYRMS9k7Ik2dn789rrb3M8+zfs/cXJPuzf8/ta+6OiIjIlfSKugAREcl8CgsREUlJYSEiIikpLEREJCWFhYiIpKSwEBGRlBQWIiKSksJCRERSUliIiEhKeVEX0FVGjhzpEydOjLoMEZEeZcuWLR+6e2GqcVkTFhMnTqSioiLqMkREehQzi6UzTruhREQkJYWFiIikpLAQEZGUFBYiIpKSwkJERFJSWIiISEoKCxERSSlrrrOQ3Haw7iyv7z7KgPzezJ5QwKSRAzGzqMsSyRoKC+mxDtWdZc2Ow6zecZhtlXWXPFcwoA8zSwqYVTKMWSUF3Dx+GAP76p+7SGfpt0d6lCMfnbsQEFtitQDcOHYIfzV/Kp+7qYjG5ha2xGrZWlnL1so63njvGAC9DK4fM4RZE+LhMaukgAkjBmj2IZImc/eoa+gSpaWlrtt9ZKejJ8/xahAQmz+IB8QNRUO4f0YRn5texKSRA9vd9qMzjWyrigfH1lgt26vqqG9oAmDEwHxmlgxjZkkBsycUMKN4KAPy9f1JcouZbXH30lTj9JshGenYyXO8uvNIEBAncIfrxwzmL++dwudmFHFt4aC0XmfogD7cOXUUd04dBUBzi/P+sVNsjdUFs49aXt8Tn3307mXcUDT4wsxjVkkB44f31+xDBM0sJIPUnGpg7c7DvPLuYTYFATF19GDuC2YQ141KLyA6qvb0+fjsIwiQd6rqOH2+GYCRg/rGj3tMiIfHjOKh9OvTO5Q6RKKQ7sxCYSGR+rC+gbU7j7D63cNs/N1xWhyuGzWI+2cUcd/0IiaPHtztNTW3OHuPnIrPPILjHx8cPwNAXi9j2tghzCopYGZw8Ly4oGfNPppbnNPnm6g/10RTc3b8/ue6/LxejBnar1PbKiwkYx2vb2DdrqOs3nGIDb+NB8Q1hQO5f8ZY7p9RxJQIAiKV4/UNbKu8uOvqnaqPONsYn32MGtw3vtsqOHh+07hwZh8tLU598CFf39DEqXNNnDrXSH3DxXUnzyV+bkwa03RhzKlzjRdmTZI9bhk/jJVP3N6pbXXMQjJK7enzrNsVPwbxn789TnOLM2nkQJ646zrum1HE1NGDM/rb+YhBfbln2mjumTYagKbmFt67ZPZRx9pdRwDo09uYNnYos0qGMTvYfTW4X167H96J9fH/tl6+GASJA/OpDOqbF3/0y2Nw8Bg7rB+D+uYxuF+f4L/xMX1667rcbDB8YH7o76GZhYSm7kwiII7wm/0f0tziTBwxgPtmFHHf9LHcUJTZAdFRNacaLsw8tsXqeKe6joamlrS3H5jfO/5hHnyQJz7o4x/+8fVD+l0MgsSH/+Dkdfl59OqVPX+nEj7NLCQSH51pZN3u+DGI3+z/kKYWp2T4AB6/4xrum17EjWOHZFVAJCsc3JfP3DiGz9w4BoDG5hb2HD7J9qo6GhpbLnzTv/jNPvig75fHwPw8eutDXjKYwkKu2kdnG1m/+yir3z3Er/d/SGOzU1zQnz/6xCTunz6Wm8Zlb0BcSZ/evZhRPIwZxcOiLkXkqikspFNOnmvk9d1HWf3uYd5+v4bGZmfcsP589fZJ3De9iBnFQ3MyIESylcJCOsTdefqlHby09SDnm1sYO7Qfj31sIp+bXsQt44cpIESylMJCOmTDgeMs31zFQzPHsfi2CdxSPEwHVEVygMJCOmRpeSVD+/fhvz40XVcyi+QQnWQtaTt28hzrdh3h4dJiBYVIjgk1LMxsvpntNbP9ZvZUG8//wMy2B499ZlaX9FyJmb1mZnvMbLeZTQyzVklt+eYqmlqcR+dOiLoUEelmoe2GMrPewLPAvUA1sNnMVrn77sQYd/9m0vivAzOTXuKnwD+4+3ozGwSkf3WTdLmm5haWbarkE5NHXvGW4CKSncKcWcwB9rv7AXc/DywHHrzC+IXAMgAzmwbkuft6AHevd/czIdYqKfzyvWMc/ugci+dpViGSi8IMi3FAVdJydbDuMmY2AZgEvBGsmgLUmdlLZrbNzL4fzFQkImXlMYqG9uNT14+KuhQRiUCYYdHW+ZTt3YhqAfCiuyduh5kHfAL4FnArcA3w2GVvYPa4mVWYWUVNTc3VVyxt+uDD0/zH+x+y4NYS8nTjOZGcFOZvfjUwPmm5GDjUztgFBLugkrbdFuzCagJWArNab+Tuz7l7qbuXFhYWdlHZ0trSjTHyehkL5oxPPVhEslKYYbEZmGxmk8wsn3ggrGo9yMymAgXAhlbbFphZIgHuBna33lbCd66xmRe2VPPpG0czekjnmquISM8XWlgEM4IngXXAHmCFu+8ys2fM7IGkoQuB5Z50r/Rgd9S3gF+a2Q7iu7T+NaxapX2r3z1M3ZlGFut0WZGcFuoV3O6+BljTat13Wi3/XTvbrgdmhFacpGVJeYxrCgdy27Ujoi5FRCKko5XSrp0HP2J7VR2L507QDQJFcpzCQtq1dGOMfn168aXZxVGXIiIRU1hIm06ea2TltkM8ePM4hvbvE3U5IhIxhYW06aUt1ZxtbNYV2yICKCykDe5O2cZKbi4eyvTioVGXIyIZQGEhlyk/cIL9x+o1qxCRCxQWcpmyjTGG9u/D528eG3UpIpIhFBZyiWMnz7Fu5xG+PFsNjkTkIoWFXOJnQYOjRXNLoi5FRDKIwkIuSDQ4+vh1I7mmcFDU5YhIBlFYyAVvvHeMQ2pwJCJtUFjIBWUbKxkzpB/33KAGRyJyKYWFABA7fpq399WwYM54NTgSkcvoU0EAWLqxkt69jIVzdGBbRC6nsJB4g6OKKj49TQ2ORKRtCgthzY7D1J5p1IFtEWmXwkIuNDj6mBociUg7FBY5bufBj9hWWcciNTgSkStQWOS4RIOjL89SgyMRaZ/CIoclGhw9cPNYhg5QgyMRaZ/CIoe9vPWgGhyJSFoUFjnK3VlSHmNG8VBmFA+LuhwRyXAKixy18XdqcCQi6VNY5Kiy8hhD+uXx+RlqcCQiqYUaFmY238z2mtl+M3uqjed/YGbbg8c+M6tr9fwQMztoZj8Ms85cc+zUOdbuPMKXZ4+nf74aHIlIanlhvbCZ9QaeBe4FqoHNZrbK3Xcnxrj7N5PGfx2Y2eplvge8FVaNuWpFosHRPN0HSkTSE+bMYg6w390PuPt5YDnw4BXGLwSWJRbMbDYwGngtxBpzTnOL8/zGSm6/bgTXqsGRiKQpzLAYB1QlLVcH6y5jZhOAScAbwXIv4L8D3w6xvpx0ocHRXB3YFpH0hRkWbd07wtsZuwB40d2bg+U/Bda4e1U74+NvYPa4mVWYWUVNTc1VlJo7yspjjB7Sl3umjY66FBHpQUI7ZkF8JjE+abkYONTO2AXAE0nLtwGfMLM/BQYB+WZW7+6XHCR39+eA5wBKS0vbCyIJxI6f5u33a/izuyfTRw2ORKQDwgyLzcBkM5sEHCQeCI+2HmRmU4ECYENinbsvSnr+MaC0dVBIxz2/sZJepgZHItJxoX29dPcm4ElgHbAHWOHuu8zsGTN7IGnoQmC5u2tmEKJzjc2sqKji3htGM2aoGhyJSMeEObPA3dcAa1qt+06r5b9L8Ro/AX7SxaXlnFd3qsGRiHSedlzniCUbYlwzUg2ORKRzFBY5YNehj9haWcejc0vo1UsNjkSk4xQWOaCsvJK+eb348mw1OBKRzlFYZLlT5xr5xfaDPHDzWIYNyI+6HBHpoRQWWe7lbQc5c14NjkTk6igsspi7s2RDjOnjhnLzeDU4EpHOU1hksU2/O8H7x+r5imYVInKVFBZZrGxjZbzB0c1qcCQiV0dhkaVqTjWwdudhvjS7WA2OROSqKSyy1IqKKhqbXQe2RaRLKCyyUKLB0ceuVYMjEekaCoss9OZ7xzhYd1azChHpMgqLLFS2McaowX25Vw2ORKSLKCyyTOXxM7y1r4YFc0rU4EhEuow+TbLM0k2xoMHR+NSDRUTSpLDIIg1NzbxQUc09N4yiaGj/qMsRkSyisMgir+44wonT53VgW0S6nMIiiywpjzFp5EBuv3Zk1KWISJZRWGSJ3YdOsiVWyyI1OBKRECgsskTZxpgaHIlIaBQWWeDUuUZWbjvI59XgSERCklZYmNnPzew+M1O4ZKCVanAkIiFL98P//wCPAu+b2T+Z2fUh1iQd4O4sKY9x07gh3Fw8NOpyRCRLpRUW7v66uy8CZgEfAOvN7D/N7Ktm1ifMAuXKNn9Qy76j8QZHZjqwLSLhSHu3kpmNAB4D/hjYBvxP4uGx/grbzDezvWa238yeauP5H5jZ9uCxz8zqgvW3mNkGM9tlZu+a2SMd/HPljLLyGIPV4EhEQpaXziAzewm4HlgCfN7dDwdP/czMKtrZpjfwLHAvUA1sNrNV7r47Mcbdv5k0/uvAzGDxDPD77v6+mY0FtpjZOnev69gfL7vVnGrg1Z2HWTR3AgPy0/pfKSLSKel+wvzQ3d9o6wl3L21nmznAfnc/AGBmy4EHgd3tjF8IfDd4zX1Jr3/IzI4BhYDCIokaHIlId0l3N9QNZjYssWBmBWb2pym2GQdUJS1XB+suY2YTgEnAZYFkZnOAfOC3bTz3uJlVmFlFTU1N6j9FFkk0OLrtmhFcN0oNjkQkXOmGxdeSdwG5ey3wtRTbtHW01dsZuwB40d2bL3kBsyLiu76+6u4tl72Y+3PuXurupYWFhSnKyS6/2qsGRyLSfdINi16WdKpNcDwi1dVf1UDyfbKLgUPtjF0ALEteYWZDgNXA37p7eZp15oyy8hiFg/vy6RvV4EhEwpduWKwDVpjZp8zsbuIf7GtTbLMZmGxmk8wsn3ggrGo9yMymAgXAhqR1+cDLwE/d/YU0a8wZVSfO8Kt9NSy8dbwaHIlIt0j3APdfA38C/Bfiu5deA350pQ3cvcnMniQeNL2BH7v7LjN7Bqhw90RwLASWu3vyLqqHgTuAEWb2WLDuMXffnma9WW3pxkoMWDCnJOpSRCRH2KWf0T1XaWmpV1S0eRZvVmloaua2f3yD0gkFPPf77Z2IJiKSHjPbcoWzWi9I9zqLycA/AtOAfon17n5NpyuUTlm7Uw2ORKT7pbvD+9+I3x+qCbgL+Cnxs5Skmy3ZEGPCiAF8/Do1OBKR7pNuWPR3918S320Vc/e/A+4Oryxpy57DJ6mI1bJ47gQ1OBKRbpXuAe5zwe3J3w8OWh8ERoVXlrSlrDxGvhociUgE0p1Z/DkwAPgzYDawGPiDsIqSy9U3NMUbHM0YS8FANTgSke6VcmYRXID3sLt/G6gHvhp6VXKZl7cd5PT5ZhbP0+myItL9Us4sgltwzE6+glu6l7tTtiHGjWOHcMv4Yak3EBHpYukes9gG/MLMXgBOJ1a6+0uhVCWXqIjVsvfoKf7poelqcCQikUg3LIYDx7n0DCgHFBbdINHg6IFb1OBIRKKRVli4u45TROTD+gbW7FCDIxGJVrpXcP8bbdxe3N3/sMsrkktcbHCkA9siEp10v6q+kvRzP+CLtH+7cekiiQZH864ZznWjBkddjojksHR3Q/08ednMlgGvh1KRXPDWvmNU157lqc9eH3UpIpLjOtsMYTKg/SIhKyuvjDc4mjYm6lJEJMele8ziFJceszhCvMeFhKTqxBne3HuMJ++6jvw8NTgSkWiluxtKO8y72fOb4g2OFqrBkYhkgLS+sprZF81saNLyMDP7Qnhl5baGpmZWbK7iUzeMZuyw/lGXIyKS9jGL77r7R4kFd68DvhtOSbJ25xGOq8GRiGSQdMOirXG6QiwkZeXxBkefUIMjEckQ6YZFhZn9s5lda2bXmNkPgC1hFpar3jtyks0f1LJobokaHIlIxkg3LL4OnAd+BqwAzgJPhFVULks0OPq92eOjLkVE5IJ0z4Y6DTwVci05r76hiZe3HuT+6UVqcCQiGSXds6HWm9mwpOUCM1sXXlm5aWWiwdFtOrAtIpkl3d1QI4MzoABw91rS6MFtZvPNbK+Z7Tezy2YmZvYDM9sePPaZWV3Sc39gZu8Hj6xv4erulJXHmFY0hJlqcCQiGSbdM5pazKzE3SsBzGwibdyFNlnQjvVZ4F6gGthsZqvcfXdijLt/M2n814GZwc/DiZ+aWxq8z5Zg29o06+1xtsRqee/IKf5RDY5EJAOlO7P4G+DXZrbEzJYAbwFPp9hmDrDf3Q+4+3lgOfDgFcYvBJYFP38GWO/uJ4KAWA/MT7PWHqmsPMbgvnk8qAZHIpKB0goLd19L/Fv+XuJnRP0l8TOirmQcUJW0XB2su4yZTQAmAW90dNtscLy+gTU7jvDQrHFqcCQiGSndGwn+MfANoBjYDswDNnBpm9XLNmtjXXu7rhYAL7p7c0e2NbPHgccBSkp67j2UVlRUc765RVdsi0jGSnc31DeAW4GYu99F/NhCTYptqoHkiwWKab9h0gIu7oJKe1t3f87dS929tLCwMEU5mam5xXl+U4y5k4YzebTu1ygimSndsDjn7ucAzKyvu78HTE2xzWZgsplNMrN84oGwqvUgM5sKFBCfqSSsAz4dnKJbAHw6WJd13t5XQ9WJs5pViEhGS3cHeXVwncVKYL2Z1ZKiraq7N5nZk8Q/5HsDP3b3XWb2DFDh7ongWAgsd3dP2vaEmX2PeOAAPOPuJ9L/Y/UcZeUxRg7qy2duVIMjEclclvQZnd4GZp8EhgJrg7OcMkJpaalXVFREXUaHVJ04wx3ff5Mn7ryOb30m1URNRKTrmdkWdy9NNa7Dp964+1udK0laW5ZocDS35x6cF5HcoH6dEWloamZFRRV3Xz+acWpwJCIZTmERkbU7j/Bh/XkWz9OsQkQyn8IiIkvLKykZPoA7JvfMU35FJLcoLCKw98gpNn1wQg2ORKTHUFhE4EKDo1I1OBKRnkFh0c1ONzTx8raD3De9iOFqcCQiPYTCoput3H6Q+oYmXbEtIj2KwqIbuTtLNsS4oWgIs0rU4EhEeg6FRTfaWhlvcLR4XokaHIlIj6Kw6EZl5ZUM6pvHF27J2tYcIpKlFBbd5Hh9A6vfPcxDs8YxsK8aHIlIz6Kw6CYvbFGDIxHpuRQW3aClxVm6McacScOZogZHItIDKSy6wVvvq8GRiPRsCotusLQ8xshB+cxXgyMR6aEUFiGrrj3DG+8d45Fbx5Ofp79uEemZ9OkVsmWbKnFg4RzdilxEei6FRYjON7Xws81VfOr6URQXDIi6HBGRTlNYhGjtrniDo0U6sC0iPZzCIkRl5THGD+/PJ9XgSER6OIVFSPYdPcWm351g0dwJanAkIj2ewiIkZeUx8nv34vdmF0ddiojIVVNYhOB0QxMvbT3I56aPYcSgvlGXIyJy1UINCzObb2Z7zWy/mT3VzpiHzWy3me0ys+eT1v+3YN0eM/tf1oPu6f2L7Yeob2jiK7fpwLaIZIfQbn9qZr2BZ4F7gWpgs5mtcvfdSWMmA08Dt7t7rZmNCtZ/DLgdmBEM/TXwSeBXYdXbVdydJeUxrh8zmFklBVGXIyLSJcKcWcwB9rv7AXc/DywHHmw15mvAs+5eC+Dux4L1DvQD8oG+QB/gaIi1dpmtlXXsOXySxfMmqMGRiGSNMMNiHFCVtFwdrEs2BZhiZr8xs3Izmw/g7huAN4HDwWOdu+8JsdYus7Q8Fm9wNFMNjkQke4TZhaetr9XexvtPBu4EioH/MLObgJHADcE6gPVmdoe7v33JG5g9DjwOUFIS/e00Tpw+zyvvHuaRW8czSA2ORCSLhDmzqAbGJy0XA4faGPMLd290998Be4mHxxeBcnevd/d64FVgXus3cPfn3L3U3UsLC6O/8O2Fiio1OBKRrBRmWGwGJpvZJDPLBxYAq1qNWQncBWBmI4nvljoAVAKfNLM8M+tD/OB2Ru+Gijc4qmTOxOFMHaMGRyKSXUILC3dvAp4E1hH/oF/h7rvM7BkzeyAYtg44bma7iR+j+La7HwdeBH4L7ADeAd5x938Pq9au8Pb7NVSeOMOiedHvDhMR6Wqh7lh39zXAmlbrvpP0swN/ETySxzQDfxJmbV2trLwy3uDoJjU4EpHsoyu4u8DBurO88d5RHi4dT9+83lGXIyLS5RQWXWDZRjU4EpHsprC4SuebWli+uYq7p45i/HA1OBKR7KSwuErrdh3hw/oGnS4rIllNYXGVyspjFBf0544p0V/nISISFoXFVXj/6Ck2Bg2OeqvBkYhkMYXFVUg0OHq4VA2ORCS7KSw6KdHg6LNqcCQiOUBh0Umr3jnEqYYmvqID2yKSAxQWneDuLNkQb3A0e4IaHIlI9lNYdMK2qjp2Hz7JIjU4EpEcobDohLLyGAPze/NFNTgSkRyhsOig2qDB0RdnjVODIxHJGQqLDnphSxXnm9TgSERyi8KiAxINjm6dWMD1Y4ZEXY6ISLdRWHTAf+z/kNjxM5pViEjOUVh0QFl5jBED1eBIRHKPwiJNh+rO8ss9R3n4VjU4EpHco7BI07JN8QZHj6rBkYjkIIVFGhINju5SgyMRyVEKizS8tvsINacaWDxPswoRyU0KizSUlccYN6w/n5wyKupSREQiobBIYf+xU5QfOMGieSVqcCQiOUthkUJZeSV9ehsPl46PuhQRkciEGhZmNt/M9prZfjN7qp0xD5vZbjPbZWbPJ60vMbPXzGxP8PzEMGtty5nzTfx8SzWfvamIkWpwJCI5LLQ74ZlZb+BZ4F6gGthsZqvcfXfSmMnA08Dt7l5rZskHBX4K/IO7rzezQUBLWLW2Z9X2oMHRbbpiW0RyW5gziznAfnc/4O7ngeXAg63GfA141t1rAdz9GICZTQPy3H19sL7e3c+EWOtl3J0l5TGmjh5MqRociUiOCzMsxgFVScvVwbpkU4ApZvYbMys3s/lJ6+vM7CUz22Zm3w9mKpcws8fNrMLMKmpqarq0+O1Vdew6dJLF80rU4EhEcl6YYdHWJ6y3Ws4DJgN3AguBH5nZsGD9J4BvAbcC1wCPXfZi7s+5e6m7lxYWFnZd5cQPbA/M780X1OBIRCTUsKgGkk8hKgYOtTHmF+7e6O6/A/YSD49qYFuwC6sJWAnMCrHWS9SePs+/v3uIL8wcx+B+fbrrbUVEMlaYYbEZmGxmk8wsH1gArGo1ZiVwF4CZjSS+++lAsG2BmSWmC3cDu+kmL26pVoMjEZEkoYVFMCN4ElgH7AFWuPsuM3vGzB4Ihq0DjpvZbuBN4Nvuftzdm4nvgvqlme0gvkvrX8OqNVm8wVGM0gkF3FCkBkciIhDiqbMA7r4GWNNq3XeSfnbgL4JH623XAzPCrK8tv97/IR8cP8Of3zOlu99aRCRj6QruVsrKYwwfmM9np6vBkYhIgsIiyaG6s7y+5ygPl6rBkYhIMoVFkuVBg6NFc3UrchGRZAqLQGNzC8s2V3HnlEI1OBIRaUVhEXht19GgwZFOlxURaU1hEUg0OLpzqhociYi0prAg3uBow4HjPDpXDY5ERNqisOBig6NHblWDIxGRtuR8WJw538TPt1YzXw2ORETalfNhcepcE5+cUsjvq8GRiEi7Qr3dR08wekg/fvhot93QVkSkR8r5mYWIiKSmsBARkZQUFiIikpLCQkREUlJYiIhISgoLERFJSWEhIiIpKSxERCQli7fB7vnMrAaIXcVLjAQ+7KJyupLq6hjV1TGqq2Oysa4J7l6YalDWhMXVMrMKdy+Nuo7WVFfHqK6OUV0dk8t1aTeUiIikpLAQEZGUFBYXPRd1Ae1QXR2jujpGdXVMztalYxYiIpKSZhYiIpKSwiJgZt8zs3fNbLuZvWZmY6OuCcDMvm9m7wW1vWxmw6KuCcDMfs/MdplZi5lFfnaImc03s71mtt/Mnoq6ngQz+7GZHTOznVHXkmBm483sTTPbE/w//EbUNSWYWT8z22Rm7wS1/X3UNSWYWW8z22Zmr0RdSzIz+8DMdgSfXRVhvY/C4qLvu/sMd78FeAX4TtQFBdYDN7n7DGAf8HTE9STsBB4C3o66EDPrDTwLfBaYBiw0s2nRVnXBT4D5URfRShPwl+5+AzAPeCKD/r4agLvd/WbgFmC+mc2LuKaEbwB7oi6iHXe5+y1hnj6rsAi4+8mkxYFARhzMcffX3L0pWCwHiqOsJ8Hd97j73qjrCMwB9rv7AXc/DywHHoy4JgDc/W3gRNR1JHP3w+6+Nfj5FPEPwHHRVhXncfXBYp/gEfnvopkVA/cBP4q6lqgoLJKY2T+YWRWwiMyZWST7Q+DVqIvIQOOAqqTlajLkwy/TmdlEYCawMdpKLgp292wHjgHr3T0TavsfwF8BLVEX0gYHXjOzLWb2eFhvklNhYWavm9nONh4PArj737j7eGAp8GSm1BWM+Rviuw+WZlJdGcLaWBf5t9FMZ2aDgJ8Df95qZh0pd28OdgcXA3PM7KYo6zGz+4Fj7r4lyjqu4HZ3n0V8N+wTZnZHGG+SF8aLZip3vyfNoc8Dq4HvhljOBanqMrM/AO4HPuXdeK5zB/6+olYNjE9aLgYORVRLj2BmfYgHxVJ3fynqetri7nVm9ivix3yiPEHgduABM/sc0A8YYmZl7r44wpoucPdDwX+PmdnLxHfLdvmxxJyaWVyJmU1OWnwAeC+qWpKZ2Xzgr4EH3P1M1PVkqM3AZDObZGb5wAJgVcQ1ZSwzM+D/AXvc/Z+jrieZmRUmzvgzs/7APUT8u+juT7t7sbtPJP5v641MCQozG2hmgxM/A58mpGBVWFz0T8EulneJ/4VnyumEPwQGA+uDU+P+b9QFAZjZF82sGrgNWG1m66KqJTgB4ElgHfGDtSvcfVdU9SQzs2XABmCqmVWb2R9FXRPxb8pfAe4O/k1tD741Z4Ii4M3g93Az8WMWGXVbT/H6AAACM0lEQVSqaoYZDfzazN4BNgGr3X1tGG+kK7hFRCQlzSxERCQlhYWIiKSksBARkZQUFiIikpLCQkREUlJYiHSAmdWnHnXF7V80s2uCnweZ2b+Y2W+DO6y+bWZzzSw/+DmnLpqVzKawEOkmZnYj0NvdDwSrfkT8JoOT3f1G4DFgZHAzxF8Cj0RSqEgbFBYinWBx3w8u5NxhZo8E63uZ2f8OZgqvmNkaM/tysNki4BfBuGuBucDfunsLQHDX3NXB2JXBeJGMoGmuSOc8RLzfws3ASGCzmb1N/OroicB0YBTxK8p/HGxzO7As+PlGYLu7N7fz+juBW0OpXKQTNLMQ6ZyPA8uCO6QeBd4i/uH+ceAFd29x9yPAm0nbFAE16bx4ECLnE/f9EYmawkKkc9q6LfqV1gOcJX7XUoBdwM1mdqXfwb7AuU7UJtLlFBYinfM28EjQqKcQuIP4jdx+DXwpOHYxGrgzaZs9wHUA7v5boAL4++AusJjZ5ESvEDMbAdS4e2N3/YFErkRhIdI5LwPvAu8AbwB/Fex2+jnx/ho7gX8h3oHuo2Cb1VwaHn8MjAH2m9kO4F+52IfjLmBNuH8EkfTprrMiXczMBrl7fTA72ES8k9mRoD/Dm8Fyewe2E6/xEvB0BvU5lxyns6FEut4rQQOffOB7wYwDdz9rZt8l3h+8sr2NgwZOKxUUkkk0sxARkZR0zEJERFJSWIiISEoKCxERSUlhISIiKSksREQkJYWFiIik9P8BFem4xl5mXNAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最高准确率为： 0.7721755368814192\n"
     ]
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "\n",
    "n_classes = 2\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "\n",
    "scores[1][:] = np.mean(lrcv_L1.scores_[1],axis = 0)\n",
    "\n",
    "pyplot.plot(np.log10(Cs), scores[1][:].reshape(n_Cs,1)) \n",
    "\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('accuracy')\n",
    "pyplot.show()\n",
    "\n",
    "print('最高准确率为：',max(scores[1][:]))\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C取0的时候为最优解"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LRL2正则："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.1, 1, 10, 100, 1000, 10000, 100000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='accuracy', solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='accuracy', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.1, 1, 10, 100, 1000, 10000, 100000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='accuracy', solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[4, 5, 4, 5, 5, 5, 5, 5],\n",
       "        [4, 5, 5, 5, 5, 5, 5, 5],\n",
       "        [4, 5, 5, 5, 5, 5, 5, 5],\n",
       "        [5, 4, 5, 5, 5, 5, 5, 5],\n",
       "        [5, 4, 5, 5, 5, 5, 5, 5]]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.n_iter_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L2正则比L1正则的迭代次数少一半"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt0lfWd7/H3F8L9fgnhEu4iEFRAgtXa2hGhg7YVbBWh0y4709bpTO3pdM7MqKtnWds5Xe2snlVn1kzPtLa1l5keISBeavEShGqtl2YjIJAAIlT3JhCu4SK3kHzPH/sJ3cQk7GTvJ89O9ue1Fos813w3Sj48v+f3fB9zd0RERNqrW9QFiIhI56YgERGRjChIREQkIwoSERHJiIJEREQyoiAREZGMKEhERCQjChIREcmIgkRERDJSEHUBHWH48OE+YcKEqMsQEelUNmzYcMjdCy+1X14EyYQJE4jFYlGXISLSqZjZO+nsp6EtERHJiIJEREQyoiAREZGMKEhERCQjChIREcmIgkRERDKiIBERkYwoSCRrduw/wX+/9g7xI6eiLkVEOlBePJAoHeP+1W/yxru1AEwtGsD8khHcNL2IWcWD6dbNIq5ORMKiIJGs2HXgBG+8W8vdN0xixIBerK2q4Ycv7uYH699meP9e3DRtBDdNH8GHpgynb0/9byfSlehvtGRFWSxBQTfjix+eROGAXnzhw5M4dqqO3+48QHllDWu27GNFLE6vgm5cf9lw5k8v4qbpIyga2Dvq0kUkQwoSyVhdfQOr30gwb9oICgf0urB+UN8eLJo1hkWzxnDufAMVfzzC2qoa1lbVsG77AXgcrioedCFUSkYNxExDYCKdTahBYmYLgX8DugM/cffvNtn+EHBjsNgXGOHug83sRuChlF2nAUvd/QkzmwgsB4YCbwCfdfdzYX4Oad267Qc4dPIcd84d2+I+PYMrkesvG84DHy9hZ83JC6Hy0NqdfL98J6MH9eam6UXMLyni2klD6VXQvQM/hYi0l7l7OCc26w7sBBYACaACWObulS3s/xVgtrv/VZP1Q4FdQLG7nzKzMmC1uy83sx8Cm939P1urpbS01NX9Nzxf+EUFbyaO8cp98yjo3vaJgAdPnGX99gOUV9Xw8luHOF1XT7+e3bnh8kLmTy/ixmkjGNqvZwiVi0hrzGyDu5dear8wr0iuAXa5++6goOXAIqDZIAGWAd9oZv3twDNBiBgwD/h0sO0XwINAq0Ei4Tlw/Azrdxzk7hsmtStEAAoH9GLJ3LEsmTuWM3X1vPL2IcorD/BCVQ3PbN1PN4M544cEQ2BFTC7spyEwkRwSZpCMAeIpywngA83taGbjgYnAumY2LwW+H3w9DKh19/Mp5xyTlWqlXVa9kaC+wVlS2vKwVlv07tGdedOKmDetiIaGK9hafYy1VQdYW1nDd57Zznee2c7E4f24adoI5pcUUTp+SLsDTESyI8wgae6fjC2Noy0FVrl7/UUnMBsFXAk819ZzmtndwN0A48aNS6deaSN3Z2UswTUThjJxeL+sn79bN+Oq4sFcVTyYv19wOXtrT7OuqobyqgP88tV3+MnLexjUpwc3Ti1kfkkRN1xeyMDePbJeh4i0LswgSQCp/0wtBqpb2Hcp8OVm1i8BHnf3umD5EDDYzAqCq5IWz+nuDwMPQ/IeSdvLl0up+ONR9hx6jy/feFmHfL8xg/vw2esm8NnrJnDy7Hl+t/Mg5VU1rN9+gCc2VVPQzbh20jBumj6C+dOLGDu0b4fUJZLvwgySCmBKMMtqL8mw+HTTncxsKjAEeLWZcywD7m9ccHc3s/Uk75ssB+4Cnsx+6ZKOslic/r0KuOXKkR3+vfv3KuDmK0dx85WjqG9w3nj3aHIWWGUN3/x1Jd/8dSXTRg64ECoz9XS9SGhCm7UFYGa3AP9KcvrvI+7+bTP7FhBz96eCfR4Eerv7fU2OnQD8Hhjr7g0p6yfxp+m/G4HPuPvZ1urQrK3sO3Gmjmu+/QKLZ4/mO5+8KupyLrLn0Hu8UFVDeWUNsXeOUt/gDO/fi/lBqFx/2XD69NTUYpFLSXfWVqhBkisUJNm3/A/vct/qLaz+2w9y9bghUZfTotpT51i/4wBrqw7w4o6DnDx7nt49uvGhywpZUDKCedOKLnqIUkT+JBem/0oXtiIWZ8qI/sweOzjqUlo1uG9PbptdzG2zizl3voHX9xxmbWVNciZYVQ1mW5g1djALSopYML2Iy0b019RikTbSFYm02Vs1J1jw0Et8/ZbpfPGGSVGX0y7uTtW+E5RXJp+u37L3GADjh/Vl/vQi5k8vYu4ETS2W/KYrEglNWSxOQTfjtqs77yM8ZkbJ6IGUjB7IV+dPYd+x07wQXKX816vv8NMmU4s/cnkhAzS1WKRZChJpk2SDxr3cNH0Ew/t3nXsLowb14TPXjucz147n5NnzvPzWQcorD7Buew1PbKqmR/fk1OIFJcmn68cM7hN1ySI5Q0EibfJC1QEOv9d6g8bOrn+vAhZeMYqFVySnFm94Jzm1uLyyhgee3MYDT26jZNRA5gf3Va4Yo67Fkt90j0Ta5PM/r2Br9TF+f2/7GjR2dm8fPBncrK9hwztHaXAYObA380uSU4uvmzxMXYuly9A9Esm6muNnWL/jAF/6yOS8DBGAyYX9mfyR/vz1RyZz+ORZ1u84yNrKGla/sZf/fu1ddS2WvKQgkbSt2pCgwclag8bOblj/Xtw+p5jb5xRzpq6eV98+THnwdH1j1+LS8UOZXzKCBSUjQ+lHJpILNLQlaXF3bvw/v2XEwN6U/fV1UZeT0xoaPNm1uDLZYLJq33EAJhf2u3BfZfa4IXRXyxbJcRrakqz6w54j/PHwKb4yb0rUpeS8i7oWf3QqiaOneKEq+e76n/5uDz96cTdD+/Vk3rQRXDNhKAXdFSgSnvklRaF3xVaQSFrKYomgUWLHN2js7IqH9OWuD07grg9O4PiZOl7ccZC1VTU8v20/qzYkoi5Puri1f/8RBYlE78SZOtZs2cfi2WPo21P/y2RiYO8efGLmaD4xczR19Q1U156OuiTp4kYNCv+ZJ/1UkEt6+s19nK6rZ0lpcdSldCk9undj/DDdgJfOLz/ncEqbrKiIc3lRf2bleINGEYmGgkRatbPmBJvitSwpHaunt0WkWQoSaVVZRZwe3Y3bZnfeBo0iEi4FibTo3PkGHt+4l/nTixjWhRo0ikh2KUikReu213D4vXN6kl1EWqUgkRaVxRKMHNibGy4vjLoUEclhChJp1v5jZ/jtjgN8as4YtfIQkVYpSKRZj72RbNB4xxwNa4lI6xQk8j7uTlkszgcmDmWCOtaKyCUoSOR9Xt9zhHcOn+rSb0EUkexRkMj7lMXiDOhVwM1XjIq6FBHpBBQkcpHjQYPGT8waTZ+eemWsiFyagkQu8vTmfZypa9CzIyKSNgWJXGRFLM7UogHMLB4UdSki0kkoSOSCHftPsDleyx2lxWrQKCJpU5DIBWWxZIPGT16t946ISPoUJAL8qUHjgpIihvbrGXU5ItKJKEgEgBeqajjy3jnu0E12EWkjBYkAyZvsIwf25oYpatAoIm2jIBH2HTvNSzsPcvucYjVoFJE2U5AIj20IGjSW6ia7iLSdgiTPNTQ4ZbEE104ayvhhatAoIm2nIMlzr+85wrtH1KBRRNov1CAxs4VmtsPMdpnZfc1sf8jMNgW/dppZbcq2cWb2vJlVmVmlmU0I1v/czPakHDcrzM/Q1a0MGjQunKEGjSLSPgVhndjMugM/ABYACaDCzJ5y98rGfdz9ayn7fwWYnXKKXwLfdvdyM+sPNKRs+0d3XxVW7fni+Jk61mzdx6euLlaDRhFptzCvSK4Bdrn7bnc/BywHFrWy/zLgUQAzKwEK3L0cwN1PuvupEGvNS7/eXK0GjSKSsTCDZAwQT1lOBOvex8zGAxOBdcGqy4FaM1ttZhvN7HvBFU6jb5vZm8HQWK8wis8HZRVxpo0cwFVq0CgiGQgzSJp7IMFb2HcpsMrd64PlAuDDwD8Ac4FJwOeCbfcD04L1Q4F7m/3mZnebWczMYgcPHmzXB+jKtu8/zubEMZaUjlWDRhHJSJhBkgBSx0yKgeoW9l1KMKyVcuzGYFjsPPAEcDWAu+/zpLPAz0gOob2Puz/s7qXuXlpYqKe1myqrSNCju7F4drMXiSIiaQszSCqAKWY20cx6kgyLp5ruZGZTgSHAq02OHWJmjQkwD6gM9h8V/G7AYmBraJ+gi0o2aEzw0ZKRatAoIhkLbdaWu583s3uA54DuwCPuvs3MvgXE3L0xVJYBy93dU46tN7N/AF4IAmMD8ONg86+CgDFgE/ClsD5DV7W2qoajp+r0JLuIZEVoQQLg7muANU3WPdBk+cEWji0Hrmpm/bwslpiXVlTEGTWoNx9Wg0YRyQI92Z5nqmtP89JbatAoItmjIMkzj21I4A53zNGzIyKSHQqSPNLQ4KzckOC6ScMYN6xv1OWISBehIMkjr+05rAaNIpJ1CpI8sjKWYEDvAhZeMTLqUkSkC1GQ5Iljp+tYs2Ufi2aNpncPNWgUkexRkOSJX2+u5ux5NWgUkexTkOSJsliyQeOVY9SgUUSyS0GSB6r2HefNxDHunKsGjSKSfQqSPFAWi9OzezcWz1KDRhHJPgVJF3f2fD1PbNzLghlFDFGDRhEJgYKki1tbeYCjp+p0k11EQqMg6eJWxOKMHtSbD102POpSRKSLUpB0YdW1p/mdGjSKSMgUJF3YqsYGjRrWEpEQKUi6qGSDxjgfnDyMsUPVoFFEwqMg6aJe232Y+JHTatAoIqFTkHRRZbE4A3oX8Ocz1KBRRMKlIOmCjp2u45mt+1k8a4waNIpI6BQkXdBTQYNGDWuJSEdQkHRBZRVxpo8ayIzRA6MuRUTyQFpBYmaPmdnHzEzBk+Mqq4+zZe8x7iwtVoNGEekQ6QbDfwKfBt4ys++a2bQQa5IMNDZoXKQGjSLSQdIKEndf6+5/AVwN/BEoN7NXzOwvzaxHmAVK+s6er+eJTXv5qBo0ikgHSnuoysyGAZ8DvgBsBP6NZLCUh1KZtFl5ZQ21atAoIh2sIJ2dzGw1MA34L+AT7r4v2LTCzGJhFSdts6IizpjBfbheDRpFpAOlFSTAf7j7uuY2uHtpFuuRdtpbe5qXdx3iK/OmqEGjiHSodIe2ppvZ4MYFMxtiZn8bUk3SDqtiQYPGOcVRlyIieSbdIPmiu9c2Lrj7UeCL4ZQkbdXYoPH6y9SgUUQ6XrpB0s1SHkows+6ApgXliFd3HyZx9LRusotIJNK9R/IcUGZmPwQc+BLwbGhVSZuUxeIMVINGEYlIukFyL/DXwN8ABjwP/CSsoiR9x04lGzQunTtWDRpFJBJpBYm7N5B8uv0/wy1H2uqpzXs5d75Bw1oiEpl0nyOZAnwHKAF6N65390kh1SVpWhGLUzJqIFeMGRR1KSKSp9K92f4zklcj54EbgV+SfDhRIrSt+hhb9x5Xu3gRiVS6QdLH3V8AzN3fcfcHgXnhlSXpWBlL0LOgG4tmjY66FBHJY+kGyZmghfxbZnaPmd0GjLjUQWa20Mx2mNkuM7uvme0Pmdmm4NdOM6tN2TbOzJ43syozqzSzCcH6iWb2upm9ZWYrzCwvpyGfqavn8Y17+fMZIxncNy//CEQkR6QbJH8H9AX+BzAH+AxwV2sHBM+a/AC4meS9lWVmVpK6j7t/zd1nufss4N+B1Smbfwl8z92nA9cAB4L1/wI85O5TgKPA59P8DF1KeWUNx07XsaRUT7KLSLQuGSRBICxx95PunnD3v3T3T7n7a5c49Bpgl7vvdvdzwHJgUSv7LwMeDb5nCVDg7uUAwfc+FTwUOQ9YFRzzC2DxpT5DV1QWCxo0TlaDRhGJ1iWDxN3rgTmpT7anaQwQT1lOBOvex8zGAxOBxsaQlwO1ZrbazDaa2feCQBsG1Lr7+TTOebeZxcwsdvDgwTaWntsSR0/x8q5D3D6nmG5q0CgiEUv3gcSNwJNmthJ4r3Glu69u+RCa+wnnLey7FFgVhFZjXR8GZgPvAitIvgvlqXTP6e4PAw8DlJaWtvR9O6VVGxIA3KFhLRHJAekGyVDgMBfP1HIuvqfRVAJInZdaDFS3sO9S4MtNjt3o7rsBzOwJ4FrgEWCwmRUEVyWtnbNLamhwVsYSXD95OMVD1KBRRKKX7pPtf9mOc1cAU8xsIrCXZFh8uulOZjYVGAK82uTYIWZW6O4HSQZYzN3dzNYDt5O853IX8GQ7auu0Xnn7MHtrT3PvzdOiLkVEBEj/yfaf0cwQkrv/VUvHuPt5M7uHZMPH7sAj7r7NzL5FMhQah6mWAcvd3VOOrTezfwBeCO7NbAB+HGy+F1huZv+b5JDbT9P5DF1FWSzOoD49+GhJUdSliIgA6Q9tPZ3ydW/gNtIYUnL3NcCaJuseaLL8YAvHlgNXNbN+N8kZYXnn2Kk6nt22n2Vq0CgiOSTdoa3HUpfN7FFgbSgVSYuebGzQqJYoIpJD0n0gsakpwLhsFiKXtqIizozRA5kxWg0aRSR3pHuP5AQX3yPZT/JehXSQrXuPsa36ON9aNCPqUkRELpLu0NaAsAuR1q2MxZMNGmc2+/yliEhk0hraMrPbzGxQyvJgM8vL1iRROFNXzxObqlk4YySD+vaIuhwRkYuke4/kG+5+rHHB3WuBb4RTkjT1/IUGjbrJLiK5J90gaW6/dKcOS4bKKpINGj84eVjUpYiIvE+6QRIzs++b2WQzm2RmD5F8SFBCFj9yit+/fYg7StWgUURyU7pB8hXgHMnmiWXAaS7ujSUh+VODRg1riUhuSnfW1nvA+95wKOFqaHBWbUjwocuGM2Zwn6jLERFpVrqztsrNbHDK8hAzey68sgTg928fYm/tad1kF5Gclu7Q1vBgphYA7n6UNN7ZLpkpiyUY3LcHH52hBo0ikrvSDZIGM7vQEsXMJtDyS6okC2pPneO5bftZPGsMvQrUoFFEcle6U3i/DrxsZi8GyzcAd4dTkgA8uak62aBRw1oikuPSvdn+rJmVkgyPTSRfJnU6zMLy3YqKOFeMGUjJ6IFRlyIi0qp0mzZ+AfgqyVfbbiL52ttXufjVu5IlW/ceo3Lfcf5ZDRpFpBNI9x7JV4G5wDvufiMwGzgYWlV5rixo0HirGjSKSCeQbpCccfczAGbWy923A1PDKyt/namr54mNe7n5CjVoFJHOId2b7YngOZIngHIzO0oar9qVtntu236Onzmvm+wi0mmke7P9tuDLB81sPTAIeDa0qvJYWSxO8ZA+XDdJDRpFpHNocwdfd3/x0ntJe8SPnOL3uw7ztfmXq0GjiHQa7X1nu4Rg5YYEZnB7aXHUpYiIpE1BkiPqG5xVsTgfnlKoBo0i0qkoSHLE73cdovrYGZboakREOhkFSY4oi8UZ3LcHC0rUoFFEOhcFSQ44+t45nt9WowaNItIpKUhywJOb9nKuXg0aRaRzUpBEzN1ZEUtw5ZhBatAoIp2SgiRiW/cep2rfcZbM1dWIiHROCpKIlcXi9Croxq0zR0ddiohIuyhIInSmrp4nNgUNGvuoQaOIdE4Kkgg9t20/J9SgUUQ6OQVJhFZUxBk7tA/XqkGjiHRiCpKIxI+c4pW3D7Nkzlg1aBSRTk1BEpGVsThm8Kk5aokiIp1bqEFiZgvNbIeZ7TKz+5rZ/pCZbQp+7TSz2pRt9SnbnkpZ/3Mz25OybVaYnyEM9Q3Oqg0JbphSyGg1aBSRTq7N7yNJl5l1B34ALAASQIWZPeXulY37uPvXUvb/Csl3wTc67e4thcQ/uvuqEMruEC8HDRq//rGSqEsREclYmFck1wC73H23u58DlgOLWtl/GfBoiPXkjLJYnCF9ezC/ZETUpYiIZCzMIBkDxFOWE8G69zGz8cBEYF3K6t5mFjOz18xscZNDvm1mbwZDY72yWnXIjr53jvJtNSyerQaNItI1hBkkzU1F8hb2XQqscvf6lHXj3L0U+DTwr2Y2OVh/PzANmAsMBe5t9pub3R0EUezgwYPt+gBheHxjskHjnWqJIiJdRJhBkgBSf1oWA9Ut7LuUJsNa7l4d/L4b+C3B/RN33+dJZ4GfkRxCex93f9jdS929tLCwMJPPkTXuTlkszlXFg5g2Ug0aRaRrCDNIKoApZjbRzHqSDIunmu5kZlOBIcCrKeuGNA5Zmdlw4HqgMlgeFfxuwGJga4ifIau27D3G9v0n9CS7iHQpoc3acvfzZnYP8BzQHXjE3beZ2beAmLs3hsoyYLm7pw57TQd+ZGYNJMPuuymzvX5lZoUkh842AV8K6zNkW2ODxk+oQaOIdCGhBQmAu68B1jRZ90CT5QebOe4V4MoWzjkviyV2mDN19Ty5qZpbrhylBo0i0qXoyfYO8uzWZIPGO0r1JLuIdC0Kkg6yoiLOuKF9uXaiGjSKSNeiIOkA7x4+xau7D7OktFgNGkWky1GQdICVG+J0U4NGEemiFCQhu9Cg8fJCRg1Sg0YR6XoUJCH73VsH2XfsjJ4dEZEuS0ESspWxBEP79WT+9KKoSxERCYWCJERH3jvH85X7WTxrDD0L9EctIl2TfrqF6PGNe6mrdzVoFJEuTUESEndnZSzOzOJBTB05IOpyRERCoyAJyZuJoEGjrkZEpItTkISkLBandw81aBSRrk9BEoLT5+p5alM1t1wxioG91aBRRLo2BUkInt22jxNnz2tYS0TygoIkBCsq4owf1pcPTBwadSkiIqFTkGTZO4ff47XdR1hSOpbkSxxFRLo2BUmWrYwlkg0ar1aDRhHJDwqSLGps0PiRywsZOah31OWIiHQIBUkWvfTWQfYfV4NGEckvCpIsWhmLM7RfT25Sg0YRySMKkiw5fPIs5ZU13DZbDRpFJL/oJ16WNDZo1LCWiOQbBUkWuDtlsTgzxw5Wg0YRyTsKkizYnDjGzpqT3KmrERHJQwqSLGhs0PjxmaOiLkVEpMMpSDJ0+lw9v95UzS1XqkGjiOQnBUmGntmabNCoYS0RyVcKkgytqIgzYVhfrlGDRhHJUwqSDPzx0Hu8vucId6hBo4jkMQVJBlZuiKtBo4jkPQVJOzU2aPyzqSPUoFFE8pqCpJ1e2nmQmuNnWVKqqxERyW8KknZaURFnWL+ezJumBo0ikt8UJO1w+ORZ1lapQaOICChI2uXxjXs53+AsmatnR0REFCRt5O6sqIgza+xgLi9Sg0YRkVCDxMwWmtkOM9tlZvc1s/0hM9sU/NppZrUp2+pTtj2Vsn6imb1uZm+Z2Qoz6xnmZ2hqU7yWtw6c5E5djYiIACEGiZl1B34A3AyUAMvMrCR1H3f/mrvPcvdZwL8Dq1M2n27c5u63pqz/F+Ahd58CHAU+H9ZnaE5ZLEGfHt35+FVq0CgiAuFekVwD7HL33e5+DlgOLGpl/2XAo62d0JKPj88DVgWrfgEszkKtaTl17jy/3pxs0DhADRpFRIBwg2QMEE9ZTgTr3sfMxgMTgXUpq3ubWczMXjOzxrAYBtS6+/lLnTMMz2zZz8mz5zWsJSKSoiDEczfXfMpb2HcpsMrd61PWjXP3ajObBKwzsy3A8XTPaWZ3A3cDjBs3Lv2qW7EiFmfi8H7MnTAkK+cTEekKwrwiSQCp/3QvBqpb2HcpTYa13L06+H038FtgNnAIGGxmjQHY4jnd/WF3L3X30sLCwvZ+hgv2HHqPP+w5wh2lxWrQKCKSIswgqQCmBLOsepIMi6ea7mRmU4EhwKsp64aYWa/g6+HA9UCluzuwHrg92PUu4MkQP8MFK2Nq0Cgi0pzQgiS4j3EP8BxQBZS5+zYz+5aZpc7CWgYsD0Ki0XQgZmabSQbHd929Mth2L/D3ZraL5D2Tn4b1GRqdr2/gsTcS3Dh1BEUD1aBRRCRVmPdIcPc1wJom6x5osvxgM8e9AlzZwjl3k5wR1mFeeivZoPGbt+omu4hIU3qyPQ0rKuIM79+Tm6aPiLoUEZGcoyC5hEMnz/JC1QFumz2GHt31xyUi0pR+Ml7C428EDRpLNawlItIcBUkr3J2yWJzZ4wYzRQ0aRUSapSBpxcbGBo26GhERaZGCpBUrY/Fkg8aZo6MuRUQkZylIWjFuaD8+d/0E+vcKdZa0iEinpp+QrfibP5scdQkiIjlPVyQiIpIRBYmIiGREQSIiIhlRkIiISEYUJCIikhEFiYiIZERBIiIiGVGQiIhIRuziFxN2TWZ2EHinnYcPJ/mu+FyjutpGdbWN6mqbrlrXeHcvvNROeREkmTCzmLuXRl1HU6qrbVRX26iutsn3ujS0JSIiGVGQiIhIRhQkl/Zw1AW0QHW1jepqG9XVNnldl+6RiIhIRnRFIiIiGVGQpMHM/tnM3jSzTWb2vJnlxCsTzex7ZrY9qO1xMxscdU0AZnaHmW0zswYzi3wmi5ktNLMdZrbLzO6Luh4AM3vEzA6Y2daoa0llZmPNbL2ZVQX/Db8adU0AZtbbzP5gZpuDur4ZdU2pzKy7mW00s6ejrqWRmf3RzLYEP7diYX4vBUl6vufuV7n7LOBp4IGoCwqUA1e4+1XATuD+iOtptBX4JPBS1IWYWXfgB8DNQAmwzMxKoq0KgJ8DC6Muohnngf/p7tOBa4Ev58if11lgnrvPBGYBC83s2ohrSvVVoCrqIppxo7vPCnsKsIIkDe5+PGWxH5ATN5bc/Xl3Px8svgYUR1lPI3evcvcdUdcRuAbY5e673f0csBxYFHFNuPtLwJGo62jK3fe5+xvB1ydI/nAcE21V4Ekng8Uewa+c+HtoZsXAx4CfRF1LVBQkaTKzb5tZHPgLcueKJNVfAc9EXUQOGgPEU5YT5MAPxs7AzCYAs4HXo60kKRg+2gQcAMrdPSfqAv4V+CegIepCmnDgeTPbYGZ3h/mNFCQBM1trZlub+bUIwN2/7u5jgV8B9+RKXcE+Xyc5JPGrXKorR1gz63LiX7K5zMz6A48Bf9fkijwy7l4fDC8XA9eY2RVR12RmHwcOuPuGqGtpxvXufjXJYd0vm9kNYX2jgrBO3Nm4+/w0d/1/wG+Ab4RYzgWXqsvM7gI+DtzkHTiXuw1/XlFLAGNTlouB6ohq6RTMrAfJEPmVu6+Oup6m3L1+k0dDAAADKklEQVTWzH5L8h5T1JMVrgduNbNbgN7AQDP7b3f/TMR14e7Vwe8HzOxxksO8ody31BVJGsxsSsrircD2qGpJZWYLgXuBW939VNT15KgKYIqZTTSznsBS4KmIa8pZZmbAT4Eqd/9+1PU0MrPCxlmJZtYHmE8O/D109/vdvdjdJ5D8f2tdLoSImfUzswGNXwMfJcTQVZCk57vBsM2bJP+D5MSUSOA/gAFAeTDF74dRFwRgZreZWQK4DviNmT0XVS3BZIR7gOdI3jguc/dtUdXTyMweBV4FpppZwsw+H3VNgeuBzwLzgv+nNgX/2o7aKGB98HewguQ9kpyZapuDioCXzWwz8AfgN+7+bFjfTE+2i4hIRnRFIiIiGVGQiIhIRhQkIiKSEQWJiIhkREEiIiIZUZCIZIGZnbz0Xq0ev8rMJgVf9zezH5nZ20Gn25fM7ANm1jP4Wg8SS05RkIhEzMxmAN3dfXew6ickGzpOcfcZwOeA4UHTyReAOyMpVKQFChKRLLKk7wUPsG4xszuD9d3M7P8GVxhPm9kaM7s9OOwvgCeD/SYDHwD+l7s3AASdi38T7PtEsL9IztAlskh2fZLk+zJmAsOBCjN7ieQT4xOAK4ERJJ+yfyQ45nrg0eDrGcAmd69v4fxbgbmhVC7STroiEcmuDwGPBp1qa4AXSf7g/xCw0t0b3H0/sD7lmFHAwXROHgTMucY+SiK5QEEikl3Nta1vbT3AaZKdYwG2ATPNrLW/m72AM+2oTSQUChKR7HoJuDN4CVMhcAPJpnkvA58K7pUUAX+WckwVcBmAu78NxIBvBp14MbMpje95MbNhwEF3r+uoDyRyKQoSkex6HHgT2AysA/4pGMp6jOS7UbYCPyL51sFjwTG/4eJg+QIwEthlZluAH/Ond6jcCKwJ9yOItI26/4p0EDPr7+4ng6uKP5B8g93+4P0a64Pllm6yN55jNXC/u+/ogJJF0qJZWyId5+ng5Uw9gX8OrlRw99Nm9g2S75J/t6WDgxdzPaEQkVyjKxIREcmI7pGIiEhGFCQiIpIRBYmIiGREQSIiIhlRkIiISEYUJCIikpH/D6dTm38pSCstAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2正则LR最高准确率为： 0.7708598590951532\n"
     ]
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 2\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "scores[1][:] = np.mean(lr_cv_L2.scores_[1],axis = 0)\n",
    "\n",
    "pyplot.plot(np.log10(Cs), scores[1][:].reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('accuracy')\n",
    "pyplot.show()\n",
    "\n",
    "print('L2正则LR最高准确率为：',max(scores[1][:]))\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C取-1的时候为最优解"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 所以综合L1和L2， L1并且C取0的时候准确率最高，为0.772,确实和gridSearch得出来的结果不一样啊。。。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Users\\SEELE\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#LinearSVC不能得到每类的概率，在Otto数据集要求输出每类的概率，这里只是示意SVM的使用方法\n",
    "#https://xacecask2.gitbooks.io/scikit-learn-user-guide-chinese-version/content/sec1.4.html\n",
    "#1.4.1.2. 得分与概率\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.8,random_state = 0)\n",
    "\n",
    "SVC1 = LinearSVC().fit(X_train_part, y_train_part)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.84      0.92      0.88       107\n",
      "          1       0.76      0.60      0.67        47\n",
      "\n",
      "avg / total       0.81      0.82      0.81       154\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[98  9]\n",
      " [19 28]]\n"
     ]
    }
   ],
   "source": [
    "#在校验集上测试，估计模型性能\n",
    "y_predict = SVC1.predict(X_val)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (SVC1, metrics.classification_report(y_val, y_predict)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_val, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "结果比LR的要好。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性SVM正则参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线性SVM LinearSVC的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_Linear(C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC2 =  LinearSVC( C = C)\n",
    "    SVC2 = SVC2.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC2.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy))\n",
    "    print(\"iterations:\",  SVC2.n_iter_)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.7987012987012987\n",
      "iterations: 11\n",
      "accuracy: 0.8051948051948052\n",
      "iterations: 23\n",
      "accuracy: 0.8116883116883117\n",
      "iterations: 170\n",
      "accuracy: 0.8116883116883117\n",
      "iterations: 1000\n",
      "accuracy: 0.8116883116883117\n",
      "iterations: 1000\n",
      "accuracy: 0.8376623376623377\n",
      "iterations: 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.7727272727272727\n",
      "iterations: 1000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmclXXd//HXGwTHfQNvjUVQScVM1BH3ZY5ZaKVZ/gxaXZK6EzNvc8tUtLoz9ZGVqYVLLpmGqcVdmFbgVqIMiigoilo6kTluKCSyfX5/fM/EOAxzDsNcc51z5v18POYx51zne53rcx4658P1XT5fRQRmZmYd6ZV3AGZmVvmcLMzMrCQnCzMzK8nJwszMSnKyMDOzkpwszMysJCcLMzMrycnCzMxKyjRZSBolaa6keZLOauf1wZKmSnpM0ixJh7fz+kJJ38gyTjMz65iyWsEtqTfwDHAo0ARMB8ZExJxWbSYAj0XEVZKGA5MjYkir128HVgAPR8SlHV2vX79+MWTIkI6amJlZGzNmzHg1IvqXardOhjGMBOZFxPMAkm4FjgTmtGoTwMbFx5sA81tekPQJ4HlgUTkXGzJkCI2NjV0QtplZzyHp7+W0y7IbagDwUqvnTcVjrY0HPiepCZgMnAwgaQPgTOCCDOMzM7MyZZks1M6xtn1eY4DrI2IgcDhwk6RepCRxWUQs7PAC0lhJjZIam5ubuyRoMzNbVZbdUE3AoFbPB9Kqm6noBGAUQEQ8JKkO6AfsBRwt6WJgU2CFpMUR8ZPWJ0fEBGACQH19vcvnmpllJMtkMR0YJmko8A9gNPCZNm1eBA4Brpe0E1AHNEfEAS0NJI0HFrZNFGZm9l5Lly6lqamJxYsXr/JaXV0dAwcOpE+fPp1678ySRUQskzQOuBvoDVwXEbMlXQg0RsQk4DTgakmnkrqojg1vsGFm1ilNTU1stNFGDBkyBGnlSEBE8Nprr9HU1MTQoUM79d5Z3lkQEZNJA9etj53X6vEcYL8S7zE+k+DMzGrM4sWLV0kUAJLYYostWJuxXa/gNjOrIW0TRanj5XKyMLMeZ/58uPHGvKOoLk4WZtajrFgBn/kMfPGL8OyzeUdTPZwszKxHufZauO++9Hjq1HxjycLq5git7dwhJwsz6zHmz4fTT4eDD4YBA2DKlLwj6lp1dXW89tprqySGltlQdXV1nX7vTGdDmZlVknHj4N13YcIEuPBCuOceiIC1HPutGAMHDqSpqandWU8t6yw6y8nCzHqEO+6AO++Eiy6CYcOgUIBf/ALmzIGdd847uq7Rp0+fTq+jKMXdUGZW8958E046CUaMgP/5n3SsUEi/a60rKitOFmZW8844A155Ba65BlqqXWyzDWy7rZNFuZwszKym3XsvXH01nHYa7LHHe19raEivL1+eR2TVxcnCzGrWO+/AiSemO4jx41d9vVBIXVSPP97toVUdJwszq1kXXgjz5qXZT+uvv+rrDQ3pt7uiSnOyMLOaNHMmXHIJHHccHHJI+2223hp23NHJohxOFmZWc5YtgxNOgH794NJLO25bKMADD8DSpd0TW7VysjCzmvPDH8Kjj8Lll8Pmm3fctlCAhQuhsbF7YqtWThZmVlOeew7OOw+OOAKOPrp0+4MOSr/dFdUxJwszqxkR8OUvwzrrwBVXlFfGo18/2HXX2iwq2JWcLMysZlx/Pfz5z3DxxbAmZZAKBfjLX6CdrautyMnCzGrCyy+nhXf77w9jx67ZuQ0NKVFMm5ZNbLXAycLMasIpp8CiRWm1dq81/GY78MB0jruiVs/Jwsyq3qRJMHEinHtuWjexpjbZBOrrPcjdEScLM6tqb70FX/0q7LJLKhjYWYVC6oZatKjrYqslmSYLSaMkzZU0T9JZ7bw+WNJUSY9JmiXp8OLxkZJmFn8el3RUlnGaWfU666y0A94110Dfvp1/n4aGtJjvwQe7LrZaklmykNQbuAI4DBgOjJE0vE2zbwETI2I3YDRwZfH4k0B9RIwARgE/k+SNmszsPR58EK66Ko1XjBy5du+1336pfLnHLdqX5Z3FSGBeRDwfEUuAW4Ej27QJYOPi402A+QAR8e+IWFY8XldsZ2b2H4sXp4qy22wD3/722r/fBhvA3nt73GJ1skwWA4CXWj1vKh5rbTzwOUlNwGTg5JYXJO0laTbwBPCVVsnDzIz//V94+mn42c9gww275j0bGmDGDFiwoGver5ZkmSzaWzvZ9g5hDHB9RAwEDgduktQLICIejoidgT2BsyXVrXIBaaykRkmN7W1Qbma16Ykn4Hvfg89/Hj7yka5730IBVqyA++/vuvesFVkmiyZgUKvnAyl2M7VyAjARICIeInU59WvdICKeAhYBH2h7gYiYEBH1EVHfv3//LgzdzCrV8uXwpS/BppvCD37Qte+9995QV+euqPZkmSymA8MkDZXUlzSAPalNmxeBQwAk7URKFs3Fc9YpHt8G2AH4W4axmlmV+MlP4JFH4Ec/SnWdutK666aBbieLVWWWLIpjDOOAu4GnSLOeZku6UNIRxWanASdKehy4BTg2IgLYH3hc0kzgTuCrEfFqVrGaWXX429/gnHPg8MNhzJhsrlEowKxZ8Kq/cd5D6bu5+tXX10ejC9Kb1awIOOywNF12zhwYPDib60ybBvvsA7fdVl6J82onaUZE1Jdq5xXcZlYVbr4Z7r47DWxnlSgglf3YaCN3RbXlZGFmFa+5Gb7+9TQA/dWvZnutddaBAw5wsmjLycLMKt6pp6YaUNdcA717Z3+9QgHmzk1lRCxxsjCzinbXXakL6pvfhJ137p5rFgrpt0t/rORkYWYV6+230zapO+0EZ5/dfdfddVfYbDN3RbXm4nxmVrG+9S1oakozoNZdt/uu26sXHHyw7yxa852FmVWkadPg8svhpJNg3327//qFArzwQvoxJwszq0BLlqSSHgMGpIKBeWhoSL99d5E4WZhZxbnoIpg9O+1VsdFG+cQwfDhsuaWTRQsnCzOrKHPmwHe+A6NHw8c+ll8cUuqKmjIlrR7v6ZwszKxirFiRNjTaaKNUKDBvDQ1prcUzz+QdSf6cLMysYvz0p/DXv8Jll6UuoLx5vcVKThZmVhFeegnOPBMOPTRtalQJttsOBg3yegtwsjCzChCRaj6tWJG2SVV7+2zmoGXcYurUFFtP5mRhZrmbOBF+97s0sD10aN7RvFdDQ9rb4skn844kX04WZpar116Dk0+GPfeEr30t72hW5fUWiZOFmeXqtNPgjTfg6qu7p6Lsmho8GLbf3uMWThZmlps//hFuuAHOOCMV76tUDQ1w332wfHnekeTHycLMcrFoUaoo+/73w7nn5h1NxwoFWLAAHnss70jy42RhZrk4//xUpO/qq6GuLu9oOtYybtGTu6KcLMys202fnhbeffnLcOCBeUdT2n/9V6oV5WSREUmjJM2VNE/SWe28PljSVEmPSZol6fDi8UMlzZD0RPF3Ics4zaz7LF2aKsputRV8//t5R1O+QiHtq7FkSd6R5COzZCGpN3AFcBgwHBgjaXibZt8CJkbEbsBo4Mri8VeBj0fELsAXgZuyitPMutell8KsWXDllbDJJnlHU75CIY2zTJ+edyT5yPLOYiQwLyKej4glwK3AkW3aBLBx8fEmwHyAiHgsIlq2Sp8N1Enqxn2yzCwLzzwDF1wARx8NR7b9NqhwBx2UVnT31K6oLJPFAOClVs+bisdaGw98TlITMBk4uZ33+RTwWES8m0WQZtY9VqyAsWNhvfXSDnjVZvPNYcQIJ4sstFfdpW1V+DHA9RExEDgcuEnSf2KStDPwfeDL7V5AGiupUVJjc3NzF4VtZlm45pq0VuHSS9N4RTUqFOChh+Cdd/KOpPtlmSyagEGtng+k2M3UygnARICIeAioA/oBSBoI3Al8ISKea+8CETEhIuojor5///5dHL6ZdZX58+H009OX7fHH5x1N5xUK8O67KWH0NFkmi+nAMElDJfUlDWBPatPmReAQAEk7kZJFs6RNgd8DZ0fEXzKM0cy6wbhxaRZRJVWU7YwDDkglSXpiV1RmySIilgHjgLuBp0iznmZLulDSEcVmpwEnSnocuAU4NiKieN72wLmSZhZ/KmArFDNbU3fcAXfemQa2t98+72jWzkYbpYKHPbGooKJGNpetr6+PxsbGvMMws1beeCMtZtt6a3jkEVhnnbwjWnvnnAMXXwyvv56SR7WTNCMi6ku18wpuM8vMGWdAc3Ma3K6FRAGp9MeyZWmBXk/iZGFmmZg6NSWJ006D3XfPO5qus+++0Ldvz+uKcrIwsy73zjtpTcV226WCgbVk/fVhn3163iC3k4WZdbkLLoB582DChPTlWmsaGuDRR9OYTE/hZGFmXeqxx9LCuxNOSOsSalGhABFpkWFP4WRhZl1m2bJUUbZfP7jkkryjyc5ee6WyJT1p3KJG5ieYWSX44Q9T98xtt8Fmm+UdTXb69oX99+9Z4xa+szCzLvHcc3Deeama7Kc+lXc02SsU4Mkn4ZVX8o6kezhZmNlai0izn/r0gSuuqO6SHuVqGY+5995cw+g2ThZmttauvz51yVx8MQxouxFBjdp9d9h4457TFeVkYWZr5eWX08K7Aw6AE0/MO5rus846af9wJwszszKcckrabvTqq6FXD/tGKRTg2WehqSnvSLLXw/7TmllXmjQJJk5MA9s77JB3NN2vZdyiJ0yhdbIws05ZsAD++79hl11SwcCeaJddYIstekZXlNdZmFmnnH12Gq+48840C6on6tULDj44JYuI2p4F5jsLM1tjDz4IV12VxitGjsw7mnwVCvDii/DCC3lHki0nCzNbI4sXp5IeQ4bAt7+ddzT5a2hIv2u9K8rJwszWyHe/C3Pnpv20N9gg72jyt+OOsNVWThZmZv/xxBNw0UXwhS/Ahz+cdzSVQUpdUVOnpnGLWuUBbqs4K1bA/ffDzJl5R2Jt3XhjKhD4gx/kHUllKRTgl7+Ep5+GnXbKO5pslJUsJN0OXAfcFRErsg3Jeqr58+GGG+Daa1NROqs8ffqkL8Uttsg7ksrSetyiRycL4CrgOODHkm4Dro+Ip7MLy3qKZctg8uS0V/PkybB8ORx0EIwfD6NGpZIKVjn69PE4RXuGDoVttkldUSedlHc02SjrTzEi/gT8SdImwBjgj5JeAq4GfhERS9s7T9Io4EdAb+CaiLiozeuDgRuATYttzoqIyZK2AH4N7ElKTOM69emsYs2bB9ddlwrQ/fOfaYDw9NPh+ONh2LC8ozNbMy3jFr/9bepGrcWyJ2V/pOIX+LHAl4DHSElgd+CPq2nfG7gCOAwYDoyRNLxNs28BEyNiN2A0cGXx+GLgXOAb5cZnle+dd+Dmm9Mt+7Bh8P3vQ309/OY3aZ76977nRGHVq6EBXn8dZs3KO5JslDtmcQewI3AT8PGI+GfxpV9JalzNaSOBeRHxfPE9bgWOBOa0ahPAxsXHmwDzASJiEfCgpO3X4LNYhZo5M3Uz3XwzvPkmbLttmn75xS/2nHLWVvtaxi2mToURI/KNJQvl9gj/JCLanUUcEfWrOWcA8FKr503AXm3ajAfukXQysAHwoTLjAUDSWGAswODBg9fkVMvYggVwyy0pScyYAeuum3ZP+9KX0phELd6mW882cCC8//1pkPvUU/OOpuuV+ye7k6RNW55I2kzSV0uc016VlLazkMeQxiQGAocDN0kq+2skIiZERH1E1Pfv37/c0ywjEfDAA+mOYeutU5G5pUvh8svTTKeWLignCqtVDQ1w331p4katKffP9sSIeLPlSUS8AZTa5qQJGNTq+UCK3UytnABMLL7nQ0Ad0K/MmKxC/OtfcMklaSXrgQemwnJf+AJMn566oMaNg803zztKs+wVCvD22+luutaUmyx6SSvrKRYHr/uWOGc6MEzSUEl9SQPYk9q0eRE4pPieO5GSRXOZMVmOli9PU10/+cl0+33GGbDllitnN/30p2nwuparcJq1dfDB6Xct7m9R7pjF3cBEST8ldSV9BfhDRydExDJJ44rn9gaui4jZki4EGiNiEnAacLWkU4vve2xEWjAv6W+kwe++kj4BfDgi5rR3Les+L7wAP/95+mlqgv79U//s8cenOwuznmzLLeEDH0jjFmedlXc0XUtRRjGT4jjCl0l3AQLuIa2bWJ5teOWrr6+PxsbVTcyytfHuu2l66zXXwJ/+lMYcPvKRNFj9sY9B31L3mGY9yCmnpC1m33gjTeyodJJmdDBR6T/KXZS3grSK+6q1Dcyqx5NPpgRx001p/vg228CFF8Kxx8KgQSVPN+uRCgX48Y/hkUfggAPyjqbrlLvOYhjwPdLiurqW4xGxbUZxWU7efht+9auUJB5+ON01HHUUnHACHHKIZzKZldIyNXzKlNpKFuX+6f+cdFexDGgAbiQt0LMaEAEPPZQSwtZbw4knwsKFcNll8I9/wK23wqGHOlGYlWPTTWG33Wpvf4tyB7jXi4g/S1JE/B0YL+kB4PwMY7OMNTenLqZrr4U5c1KBuDFj0ljEyJGeyWTWWYUC/OhH8O9/w/rr5x1N1yj334qLi4Pcz0oaJ+koYMsM47KMrFgB99wDxxyTSm2cdhpssknqdvrnP9PA3F57OVGYrY1CAZYsgb/+Ne9Iuk65dxZfB9YHvgZ8m9QV9cWsgrKu99JLabrrddfB3/+e9iMYNy51Pe28c97RmdWW/fdP5fWnTIEPrVERo8pVMlkUF+AdExGnAwtJ+1pYFViyBP7v/9Jdw913p2OHHppWWx9xRHVM6zOrRhtumLpya2lxXslkERHLJe1RHK+o4R1ma8dTT6VxiBtvTOMSgwbBuefCccfBkCF5R2fWMxQKqez+W2/BxhuXbl/pyu2Gegz4bXGXvEUtByPijkyisjW2aBHcdlu6i/jLX9It8JFHpsHqQw+F3r3zjtCsZ2logO98JxXX/OhH845m7ZWbLDYHXgMKrY4F4GSRowhobEwJ4pZb0hqJHXaASy+Fz38+lR4ws3zss0/q6p0ypQcli4jwOEUFef11+MUvUlfTrFlpat4xx6S7iH339Uwms0qw3nrp77FWxi3KXcH9c1bdi4KIOL7LI7J2rVgB996b7iLuuCPVa9pzT/jZz2D06NroEzWrNQ0NcP758NpraQZiNSt3ncXvgN8Xf/5Mqga7MKugbKV//CNtQbr99qncxh/+AGPHpn0iHnkkPXaiMKtMhULqLr7vvrwjWXvldkPd3vq5pFuAP2USkbF0Kfz+96mbafLkdFdRKKSkcdRRUFdX+j3MLH977pkqI0ydmvZ+qWblDnC3NQzwptdd7NlnU4K4/vq0+9z73gdnn52mvG63Xd7Rmdma6ts3FROshTpR5Y5ZvM17xyxeBs7MJKIe5t//httvT2MR99+fprh+7GNpsHrUqDQF1syqV0MDnHkmvPwybLVV3tF0XrndUBtlHUhP8+ij6S7i5pthwYI0JnHRRWnv6q23zjs6M+sqheKCg3vvTZNRqlVZA9ySjpK0Savnmxa3OrU18OabcOWVsPvusMceqU7Txz+e/id65pn0rw8nCrPasttuqVhntXdFlTsb6vyIWNDyJCLexOXJy9IyE+Lzn0+J4KST0rErrkhVXm+6KW2W4rURZrWpd+/0N17tyaLcHvH2kop70zvw8stwww2pq+nZZ9O/LI47Lo1F7L573tGZWXcqFGDSJHjxRRhcpVODyr2zaJT0A0nbSdpW0mXAjCwDq0bLlsHvfgef+AQMHAhnnZXuJm68EebPX9kFZWY9S8u4RTWv5i43WZwMLAF+BUwE3gFOKnWSpFGS5kqaJ+msdl4fLGmqpMckzZJ0eKvXzi6eN1fSR8qMMxfPPQfnnAPbbJPGIKZNg298A+bOXdkFVSu7ZZnZmtt5Z+jXr7q7osqdDbUIWOXLviPFfTCuAA4FmoDpkiZFxJxWzb4FTIyIqyQNByYDQ4qPRwM7A+8D/iTp/RGxfE1iyNLixXDnnWnK65QpaX/qww5LYxEf/Sj06ZN3hGZWKXr1SlNop0xJY5bVOEZZ7myoP0ratNXzzSTdXeK0kcC8iHg+IpYAtwJHtmkTpNIhAJsA84uPjwRujYh3I+IFYF7x/XI3axZ87WtpwdxnPgMvvJDKEL/44souKCcKM2urUICmptQTUY3KHaTuV5wBBUBEvCGpVAHsAcBLrZ43AXu1aTMeuEfSycAGQMsGhAOAaW3OHVBmrF3urbdSCfBrr4Xp09OqzE9+Mg1WNzSkfzWYmXWkZdxiypS0rqralPs1t0LSf8bwJQ2hnSq0bbR3o9X2nDHA9RExEDgcuElSrzLPRdJYSY2SGpubm0uEs2Yi0iZCxx2XBqm/8hV45x340Y/SYPUtt6TCfk4UZlaOYcNSj0S1jluUe2dxDvCgpJbaiQcCY0uc0wQMavV8ICu7mVqcAIwCiIiHJNUB/co8l4iYAEwAqK+v75ItX195Jc1euuaaNEC94Ybw2c+mu4g996zOvkYzy5+U7i7uuac6xy3K+ndxRPwBqAfmkmZEnUaaEdWR6cAwSUMl9SUNWE9q0+ZF4BAASTsBdUBzsd1oSetKGkoqXPhIWZ+oE5Yvh7vugqOPhgED4PTTU+35665LC+cmTEibr1fbf1wzqyyFQvoH6Zw5pdtWmnILCX4JOIX0L/yZwN7AQ7x3m9X3iIhlksYBdwO9gesiYrakC4HGiJhESjpXSzqV1M10bEQEMFvSRGAOsAw4KauZUM8/n1ZXNjWlqW2nnAInnAA77ZTF1cysJ2toSL+nTEnTaauJ0ndziUbSE8CewLSIGCFpR+CCiPh01gGWq76+PhobG9f4vOXL4dhj4cgj4Ygj0uC1mVlWtt0WRoxIO15WAkkzIqK+VLtyxywWR8RiSUhaNyKelrTDWsZYEXr3TvWZzMy6Q6GQEsXy5en7p1qUO5enqbjO4jfAHyX9lnYGnM3MrGMNDfDGG/D443lHsmbKXcF9VPHheElTSQvo/pBZVGZmNar1uEU11Ypb41UCEXFfREwqrso2M7M18L73wY47Vl9RQS8pMzPrZg0NaRvlpUvzjqR8ThZmZt2sUICFC6ETEzhz42RhZtbNDj44/a6mrignCzOzbtavH+y6a3XViXKyMDPLQUNDKlb67rt5R1IeJwszsxwUCmkTtWnTSretBE4WZmY5OPDAtMVBtXRFOVmYmeVgk01gjz2cLMzMrIRCAR5+GBYtyjuS0pwszMxyUiikhXl/+UvekZTmZGFmlpP99oM+faqjK8rJwswsJxtsAHvt5WRhZmYlFAowYwYsWJB3JB1zsjAzy1FDA6xYkQoLVjInCzOzHO29N9TVVX5XlJOFmVmO6urSQHelFxV0sjAzy1mhkLZZffXVvCNZPScLM7OctWy1eu+9uYbRoUyThaRRkuZKmifprHZev0zSzOLPM5LebPXa9yU9Wfz5dJZxmpnlqb4eNtywsrui1snqjSX1Bq4ADgWagOmSJkXEnJY2EXFqq/YnA7sVH38U2B0YAawL3Cfproh4K6t4zczy0qdPKixYyYPcWd5ZjATmRcTzEbEEuBU4soP2Y4Bbio+HA/dFxLKIWAQ8DozKMFYzs1w1NMDTT8P8+XlH0r4sk8UA4KVWz5uKx1YhaRtgKNCSVx8HDpO0vqR+QAMwqJ3zxkpqlNTY3NzcpcGbmXWnQiH9rtSuqCyThdo5FqtpOxr4dUQsB4iIe4DJwF9JdxsPActWebOICRFRHxH1/fv375qozcxysOuusNlmPTNZNPHeu4GBwOpusEazsgsKgIj4bkSMiIhDSYnn2UyiNDOrAL17w0EHVe64RZbJYjowTNJQSX1JCWFS20aSdgA2I909tBzrLWmL4uMPAh8E7skwVjOz3BUK8MIL8Le/5R3JqjJLFhGxDBgH3A08BUyMiNmSLpR0RKumY4BbI6J1F1Uf4AFJc4AJwOeK72dmVrMqedxC7/2Orl719fXR2NiYdxhmZp0WAVttBR/+MNx0U/dcU9KMiKgv1c4ruM3MKoSUptBOmZISRyVxsjAzqyCFQlpr8WyFTelxsjAzqyAt4xaVNivKycLMrIJstx0MHOhkYWZmHZDS3cXUqWkHvUrhZGFmVmEKhbS3xezZeUeykpOFmVmFadnfopK6opwszMwqzODBaezCycLMzDpUKMB998Hy5XlHkjhZmJlVoIYGWLAAHnss70gSJwszswpUaeMWThZmZhVoq61g+PDKKSroZGFmVqEKBXjgAViyJO9InCzMzCpWQwMsWgTTp+cdiZOFmVnFOuigtKK7EsYtnCzMzCrUFlvAiBGVMW7hZGFmVsEaGuCvf4V33sk3DicLM7MKVijAu+/CQw/lG4eThZlZBTvgAOjdO/+uKCcLM7MKtvHGUF+f/yC3k4WZWYUrFOCRR2DhwvxiyDRZSBolaa6keZLOauf1yyTNLP48I+nNVq9dLGm2pKck/ViSsozVzKxSFQqwbBk8+GB+MWSWLCT1Bq4ADgOGA2MkDW/dJiJOjYgRETECuBy4o3juvsB+wAeBDwB7AgdlFauZWSXbd1/o0yffrqgs7yxGAvMi4vmIWALcChzZQfsxwC3FxwHUAX2BdYE+wL8yjNXMrGKtvz7ss0/tJosBwEutnjcVj61C0jbAUGAKQEQ8BEwF/ln8uTsinsowVjOzilYopHLlb7yRz/WzTBbtjTHEatqOBn4dEcsBJG0P7AQMJCWYgqQDV7mANFZSo6TG5ubmLgrbzKzyFAqwYgXcf38+188yWTQBg1o9HwjMX03b0azsggI4CpgWEQsjYiFwF7B325MiYkJE1EdEff/+/bsobDOzyjNyJKy3Xn5dUVkmi+nAMElDJfUlJYRJbRtJ2gHYDGi9PvFF4CBJ60jqQxrcdjeUmfVY664L++9fg8kiIpYB44C7SV/0EyNitqQLJR3RqukY4NaIaN1F9WvgOeAJ4HHg8Yj4v6xiNTOrBoUCPPkkvPJK9197nSzfPCImA5PbHDuvzfPx7Zy3HPhylrGZmVWblq1W770Xjjmme6/tFdxmZlVijz1go43y6YpysjAzqxLrrJM2RMqjqKCThZlZFWlogGeegaam7r2uk4WZWRUpFNIungE9AAAHXUlEQVTv7r67cLIwM6siH/wgbL65k4WZmXWgVy84+GD4858hVlcTI4vrdt+lzMysKxQK8OKL8MIL3XdNJwszsyrTMm7RnVNonSzMzKrMjjvCVlt177iFk4WZWZWR0hTaKVO6b9zCycLMrAoVCvDyy/D0091zPScLM7Mq1N3rLZwszMyq0NChMHhw9w1yO1mYmVUhKd1dTJ2adtDLmpOFmVmVKhTg9dfhiSeyv5aThZlZlWrZ36I7uqKcLMzMqtTAgTBsWPcki0x3yjMzs2wdeywsWpT9dZwszMyq2De/2T3XcTeUmZmV5GRhZmYlOVmYmVlJmSYLSaMkzZU0T9JZ7bx+maSZxZ9nJL1ZPN7Q6vhMSYslfSLLWM3MbPUyG+CW1Bu4AjgUaAKmS5oUEXNa2kTEqa3anwzsVjw+FRhRPL45MA+4J6tYzcysY1neWYwE5kXE8xGxBLgVOLKD9mOAW9o5fjRwV0T8O4MYzcysDFkmiwHAS62eNxWPrULSNsBQoL2lJaNpP4kgaaykRkmNzc3NaxmumZmtTpbJQu0cW902HaOBX0fE8ve8gbQ1sAtwd3snRcSEiKiPiPr+/fuvVbBmZrZ6WS7KawIGtXo+EJi/mrajgZPaOX4McGdELC11sRkzZrwq6e9rHOVK/YBX1+L8SlErnwP8WSpVrXyWWvkcsHafZZtyGmWZLKYDwyQNBf5BSgifadtI0g7AZsBD7bzHGODsci4WEWt1ayGpMSLq1+Y9KkGtfA7wZ6lUtfJZauVzQPd8lsy6oSJiGTCO1IX0FDAxImZLulDSEa2ajgFujXjvTrKShpDuTO7LKkYzMytPprWhImIyMLnNsfPaPB+/mnP/xmoGxM3MrHt5BfdKE/IOoIvUyucAf5ZKVSufpVY+B3TDZ1Gb3h8zM7NV+M7CzMxKcrIokvRtSbOKtajukfS+vGPqLEmXSHq6+HnulLRp3jF1lqT/J2m2pBWSqm7mSqn6aNVE0nWSXpH0ZN6xrA1JgyRNlfRU8f+tU/KOqbMk1Ul6RNLjxc9yQWbXcjdUImnjiHir+PhrwPCI+ErOYXWKpA8DUyJimaTvA0TEmTmH1SmSdgJWAD8DvhERjTmHVLZifbRnaFUfDRjTuj5aNZF0ILAQuDEiPpB3PJ1VXOy7dUQ8KmkjYAbwiWr87yJJwAYRsVBSH+BB4JSImNbV1/KdRVFLoijagNWvNq94EXFPceoywDTSgsiqFBFPRcTcvOPopDWtj1bRIuJ+4PW841hbEfHPiHi0+Pht0tT+qpx5GcnC4tM+xZ9MvrucLFqR9F1JLwGfBc4r1b5KHA/clXcQPVTZ9dEsH8X1XLsBD+cbSedJ6i1pJvAK8MeIyOSz9KhkIelPkp5s5+dIgIg4JyIGATeTFhRWrFKfpdjmHGAZ6fNUrHI+S5Vak/po1s0kbQjcDny9Tc9CVYmI5RExgtSDMFJSJl2EmS7KqzQR8aEym/4S+D1wfobhrJVSn0XSF4GPAYe0XR1fadbgv0u1WZP6aNaNiv37twM3R8QdecfTFSLiTUn3AqOALp+E0KPuLDoiaVirp0cAT+cVy9qSNAo4EzjC+4Dk6j/10ST1JdVHm5RzTD1ecVD4WuCpiPhB3vGsDUn9W2Y7SloP+BAZfXd5NlSRpNuBHUgzb/4OfCUi/pFvVJ0jaR6wLvBa8dC0Kp7ZdRRwOdAfeBOYGREfyTeq8kk6HPgh0Bu4LiK+m3NInSbpFuBgUoXTfwHnR8S1uQbVCZL2Bx4AniD9vQN8s1ieqKpI+iBwA+n/r16kGnwXZnItJwszMyvF3VBmZlaSk4WZmZXkZGFmZiU5WZiZWUlOFmZmVpKThdkakLSwdKsOz/+1pG2LjzeU9DNJzxUrht4vaS9JfYuPe9SiWatsThZm3UTSzkDviHi+eOgaUmG+YRGxM3As0K9YdPDPwKdzCdSsHU4WZp2g5JJiDasnJH26eLyXpCuLdwq/kzRZ0tHF0z4L/LbYbjtgL+BbEbECoFid9vfFtr8ptjerCL7NNeucTwIjgF1JK5qnS7of2A8YAuwCbEkqf31d8Zz9gFuKj3cmrUZfvpr3fxLYM5PIzTrBdxZmnbM/cEux4ue/gPtIX+77A7dFxIqIeBmY2uqcrYHmct68mESWFDfnMcudk4VZ57RXfryj4wDvAHXFx7OBXSV19De4LrC4E7GZdTknC7POuR/4dHHjmf7AgcAjpG0tP1Ucu/gvUuG9Fk8B2wNExHNAI3BBsQoqkoa17OEhaQugOSKWdtcHMuuIk4VZ59wJzAIeB6YAZxS7nW4n7WPxJGnf8IeBBcVzfs97k8eXgK2AeZKeAK5m5X4XDUDVVUG12uWqs2ZdTNKGEbGweHfwCLBfRLxc3G9gavH56ga2W97jDuDsKt5/3GqMZ0OZdb3fFTek6Qt8u3jHQUS8I+l80j7cL67u5OJGSb9xorBK4jsLMzMryWMWZmZWkpOFmZmV5GRhZmYlOVmYmVlJThZmZlaSk4WZmZX0/wHM0e66q2/ywwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-3, 3, 7)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份  \n",
    "#penalty_s = ['l1','l2']\n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "#    for j, penalty in enumerate(penalty_s):\n",
    "    tmp = fit_grid_point_Linear(oneC, X_train, y_train, X_val, y_val)\n",
    "    accuracy_s.append(tmp)\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "#for j, penalty in enumerate(penalty_s):\n",
    "pyplot.plot(x_axis, np.array(accuracy_s), 'b-')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuracy' )\n",
    "\n",
    "pyplot.show()\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线性SVM最佳参数是C = 2，最高准确率为0.837，迭代次数1000，很高了。随着C值的提升，模型复杂度在提升。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy))\n",
    "    print(\"support_vectors_ :\", SVC3.support_vectors_ )\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.7467532467532467\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.8246753246753247\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [-1.14185152  0.04334517 -0.02442979 ...  0.56020242 -0.64593181\n",
      "   1.59617091]\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.6948051948051948\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.8051948051948052\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [-0.84488505  0.20773128  1.29800761 ...  0.58931073  1.76714301\n",
      "   0.31985461]\n",
      " [-1.14185152  0.04334517 -0.02442979 ...  0.56020242 -0.64593181\n",
      "   1.59617091]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.8636363636363636\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " [-0.84488505 -0.6141993  -3.49582796 ...  1.57899345 -0.87244072\n",
      "  -0.0204964 ]\n",
      " ...\n",
      " [-1.14185152  0.04334517 -0.02442979 ...  0.56020242 -0.64593181\n",
      "   1.59617091]\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.987012987012987\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.8116883116883117\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [-0.84488505  0.20773128  1.29800761 ...  0.58931073  1.76714301\n",
      "   0.31985461]\n",
      " [-1.14185152  0.04334517 -0.02442979 ...  0.56020242 -0.64593181\n",
      "   1.59617091]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.9155844155844156\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " [-0.84488505 -0.6141993  -3.49582796 ...  1.57899345 -0.87244072\n",
      "  -0.0204964 ]\n",
      " ...\n",
      " [ 1.23388019  1.06253908  0.47148423 ... -0.00740973 -0.08720986\n",
      "   1.00055664]\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.8181818181818182\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 0.04601433 -0.38405874  1.62861696 ...  0.74940647 -0.84827977\n",
      "  -0.27575966]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [-1.14185152  0.04334517 -0.02442979 ...  0.56020242 -0.64593181\n",
      "   1.59617091]\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 0.9415584415584416\n",
      "support_vectors_ : [[ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [-0.84488505 -0.6141993  -3.49582796 ...  1.57899345 -0.87244072\n",
      "  -0.0204964 ]\n",
      " [-0.25095213  0.14197684  1.29800761 ...  0.99682715  0.70104112\n",
      "  -0.53102292]\n",
      " ...\n",
      " [ 1.23388019  1.06253908  0.47148423 ... -0.00740973 -0.08720986\n",
      "   1.00055664]\n",
      " [-0.84488505  0.20773128  1.29800761 ...  0.58931073  1.76714301\n",
      "   0.31985461]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [ 1.82781311 -0.21967262 -0.18973447 ...  0.41466084 -1.02042653\n",
      "  -0.36084741]\n",
      " [ 1.82781311  0.56938074  0.63678891 ... -0.77878009  2.92686858\n",
      "   2.02160968]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n",
      "accuracy: 1.0\n",
      "support_vectors_ : [[-0.84488505 -1.20598931 -0.52034382 ... -0.85155088 -0.36506078\n",
      "  -0.19067191]\n",
      " [-0.84488505 -1.07448042 -0.52034382 ... -0.63323851 -0.92076261\n",
      "  -1.04154944]\n",
      " [ 0.3429808  -0.1867954   0.14087488 ... -0.99709246 -0.81807858\n",
      "  -0.27575966]\n",
      " ...\n",
      " [ 0.63994726  2.24611912  1.62861696 ...  0.44376915 -0.58552944\n",
      "   2.78739945]\n",
      " [ 1.53084665  1.58857465  0.14087488 ...  1.68087256 -0.20801461\n",
      "   0.83038113]\n",
      " [-0.84488505  0.14197684 -1.01625784 ... -0.34215536 -0.37110101\n",
      "   1.17073215]]\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 2, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-2, 2, 5)  \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_val, y_val)\n",
    "        accuracy_s.append(tmp)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述部分运行结果来看，出现了准确率为1的情况，参数不错。但是参数里没有迭代次数，不知道应该从哪个参数观察模型复杂度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-2. -1.  0.  1.  2.] ^^^^ [0.69480519 0.74675325 0.80519481 0.81168831 0.81818182] ^^^^ 0.01\n",
      "[-2. -1.  0.  1.  2.] ^^^^ [0.69480519 0.82467532 0.86363636 0.91558442 0.94155844] ^^^^ 0.1\n",
      "[-2. -1.  0.  1.  2.] ^^^^ [0.69480519 0.69480519 0.98701299 1.         1.        ] ^^^^ 1.0\n",
      "[-2. -1.  0.  1.  2.] ^^^^ [0.69480519 0.69480519 1.         1.         1.        ] ^^^^ 10.0\n",
      "[-2. -1.  0.  1.  2.] ^^^^ [0.69480519 0.69480519 1.         1.         1.        ] ^^^^ 100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8VFX6+PHPSa+QRhJICKEkQOi9KkURFCkCLsKC6FrWtfx2XXX1u2vZZdVFV3fF1XUtoCgKAiKgoghIlRqQGkISIJACAUJII2Umc35/3EkIEMhAZjIpz/v1yovMrU+G5D5z7rnnOUprjRBCCHEtLs4OQAghRN0nyUIIIUS1JFkIIYSoliQLIYQQ1ZJkIYQQolqSLIQQQlRLkoUQQohqSbIQQghRLUkWQgghquXm7ADsJSQkREdHRzs7DCGEqFd27dp1VmvdrLrtGkyyiI6OJj4+3tlhCCFEvaKUOm7LdnIbSgghRLUkWQghhKiWJAshhBDVkmQhhBCiWpIshBBCVMthyUIpNVcpdVopdeAq65VS6m2lVIpSap9SqmeldTOUUsnWrxmOilEIIYRtHNmy+AQYdY31twMx1q+HgfcAlFJBwEtAP6Av8JJSKtCBcQohhKiGw8ZZaK03KqWir7HJOOBTbczruk0pFaCUag4MBVZrrc8BKKVWYySdBY6KVQhHOvDxas4cyXZ2GHWWRVsotZgwlZkwWUopLSvFoi3ODqte8Qx0Z9JLTzj0HM4clBcBpFV6nW5ddrXlV1BKPYzRKiEqKsoxUQpRA/knTrNxq0a7hIJcAIWDeJ1Pq36jGnJmslBVLNPXWH7lQq0/AD4A6N27d5XbCOFM+z/fhHYJZOKMcMIHxDk7HLsqNhdz+sJpThWeIutCFlkXsozvC7MqXp8rPnfFfk08mhDuG06YTxhhvmGE+4QT5htGmE9YxXIfdx8n/ETiWpyZLNKBlpVeRwKZ1uVDL1u+vtaiEsJOLBYLySllBKgswgcMd3Y416XIXHTJRf/yJHCq8BTnS85fsV+AZ0BFEugS0qUiCZQnhVCfUEkE9ZQzk8UK4HGl1EKMzuxcrfVJpdQq4NVKndq3Af/nrCCFuFHpa3+hwD2Evu3znR3KJS6YLlwzCWRdyCK3JPeK/QI9Aysu/t2adbuiZRDqE4q3m7cTfiJRGxyWLJRSCzBaCCFKqXSMJ5zcAbTW/wNWAncAKcAF4H7runNKqb8DO62Hmlne2S1EfbL/u0O4lIXQefrQWjvnBdMFTl04dUkSqHybKKswi7zSvCv2C/IKIswnjBZ+LegR2qPidlD5v6E+oXi5edXazyHqHkc+DTWlmvUaeOwq6+YCcx0RlxC1oSS3gLTCICK8zuId0tQuxyw0FVaZBE5dsC4rzCLfdGUrJsgriHDfcCL9IukV2stIAJX6CEJ9QvF09bRLjKLhajAlyoWoSw7O30CZqzedR7S1afv80vyrtgTKXxeYCq7YL8Q7hDCfMKL8o+gT3udix3GlRODh6mHvH080QpIshHCAw3ty8bEUEX3nBErKSjiRd+KaTw0Vmgov2V+hKhJBdNNo+rfof0kSCPMNI9Q7FHdXdyf9hKKxkWQhhJ2d+SWFc67hdInMxoKFe769h5TzKRXrFYpm3s0I8w2jbUBbBrYYeEkSCPMJo5l3M0kEok6RZCGEne1bvBN0M7pOHcim9E2knE/h0e6PMqD5AMJ8wgjxCcHdRRKBqF8kWQhhR2UlJo5leRPmdoqAmFtZvOYVQrxDeLDLg5IgRL0mJcqFsKOkrzZT4t6EDgPCyCzIZHPGZu5qd5ckClHvSbIQwo4ObcrA3VxAh1/dxNLkpQBMip3k5KiEqDlJFkLYSf7xU5wqCyM6qAC83Pg6+WsGRQyihV8LZ4cmRI1JshDCTvZ9/jPaxZWuE3qwIX0Dp4tOc3fs3c4OSwi7kGQhhB1YLBZSjpQRYM4ivH9HFictJtQnlJsjb3Z2aELYhSQLIewgbc1uCtxDiO3kQ0ZBBlsytjAhZgJuLvLAoWgYJFkIYQcHVibiUlZK5+lD+SrpK5RSTIyZ6OywhLAbSRZC1FBJTj5phUFEep3FLciHr1O+5qaImwj3DXd2aELYjSQLIWro4OcbKXP1otPIdqxPW8/ZorPSsS0aHEkWQtTQ4b25+JjOEX1HXxYfXkyYTxiDIwY7Oywh7EqShRA1cGZ3Mudcw2nXSpNRkMHWk1uZGDMRVxdXZ4cmhF1JshCiBvYtjkfpMrr+eiBLkpfgoly4K+YuZ4clhN1JshDiBpWVmDh22odQlYVPdCjLUpZxc+TN0rEtGiR5CFyIG5S0ZBMl7v50HODLT2k/ca74nHRsiwZLWhZC3KBDmzPxMBXQfvJNLE5aTHPf5gxqMcjZYQnhEJIshLgBeaknjaKBIQVklmax/eR26dgWDZokCyFuwH5r0cAuE3uyJGkJrspVOrZFgybJQojrZLFYSD6qCTBnEdSrDctSljG05VBCfUKdHZoQDuPQZKGUGqWUOqyUSlFKPVfF+lZKqbVKqX1KqfVKqchK68qUUnusXyscGacQ1yNt9W4K3YOJ7ezL2hNrySnJkY5t0eA5LFkopVyBd4HbgThgilIq7rLN3gA+1Vp3BWYC/6i0rkhr3d36NdZRcQpxvQ58by0aOG0oi5MWE+EXwYAWA5wdlhAO5ciWRV8gRWt9VGtdCiwExl22TRyw1vr9uirWC1GnVBQN9D7LKfdsdp7ayaTYSbgouaMrGjZH/oZHAGmVXqdbl1W2Fyiv43wX4K+UCra+9lJKxSultimlxjswTiFsdmD+Bspcveg8MoYlSUtwU26Mbye/nqLhc2SyUFUs05e9fhoYopT6BRgCZABm67oorXVvYCrwllKq7RUnUOpha0KJP3PmjB1DF6JqSfvy8DGdI/y2riw/spxhUcMI8Q5xdlhCOJwjk0U60LLS60ggs/IGWutMrfUErXUP4C/WZbnl66z/HgXWAz0uP4HW+gOtdW+tde9mzZo55IcQotzpXUlG0cBozdq0teSW5DIpdpKzwxKiVjgyWewEYpRSrZVSHsA9wCVPNSmlQpSquNn7f8Bc6/JApZRn+TbAICDBgbEKUa19S3YZRQOnDmTx4cVE+kXSv3l/Z4clRK1wWLLQWpuBx4FVwCFgkdb6oFJqplKq/OmmocBhpVQSEAa8Yl3eEYhXSu3F6PiepbWWZCGcpnLRwOzgEnaf3i0d26JRcWghQa31SmDlZcterPT9EmBJFfttAbo4MjYhrsfhxZsodfen40A/Fictxs1FOraFE2kNeRmQtgPS48HDB4Y/79BTStVZIWxw6OdMPEz+RE8YyopvXuSWqFsI9g6ufkch7MFUDCf3WJPDTuMr/6Sxzs0LYkc5PARJFkJUIy/1JFllYcQ0O8NPWRvJK82TEdvCcbSG8ycuJoW0HXBqP1hMxvqAVhA9GCL7GF/hXcDV3eFhSbIQohr75/+Mdgmi68SePJP0Kq2atKJveF9nhyUaitILkPnLxeSQvhMKsox17j7QoicMeAxa9jWSg59zapBJshDiGiwWC8nHNIEqi/x2UfyS8AtP9XoKpaoaRiRENbSGnGNGP0P5LaWsA2CxDi8LagNthhpJoWVfCO0ErnXjMl03ohCijjqxaheF7sF07ljA4qTFuLu4M66dVKURNiopsLYarB3R6Tuh0DqA2MMPInrCoN9DZF+I7A2+dXeApyQLIa7hwA+HcS0Lpt2Ufvy/NbO4tdWtBHoFOjssURdpDeeOVuqE3gFZB0FbjPXB7aDdCGhp7WsIjYN6NFmWJAshrqLkXB7pF4KI8M5mfd5W8k350rEtLirJh4xd1k5oa19D0TljnYc/RPaCm562dkT3Bp8g58ZbQ5IshLiKA/M3UubqQ+dRsTyfNJvWTVvTO6y3s8MSzmCxQHaK9XaSNTmcTqCi3F1Ie+hwhzUx9IVm7etVq8EWkiyEuIrD+/Lw1UUU923Ovu/28UzvZ6Rju7EozrX2McRf7G8oPm+s82xqtBQ6jjFuKUX0Au+Gf2tSkoUQVcjaeZgct3C6RWTzVcpXeLh4MLatzMHVIFkscPbwxTEN6fFwJhGj1aAgtCPEjbV2QveBkFhwaXxlXiRZCFGF/Ut3o3QIsff04pmd/2JE9AgCvAKcHZawh6IcSN918ZZS+i4oyTXWeQUYCaHzBKP1ENELvJo6N946QpKFEJcpLxoY5p7FzxRQYCqQju36ylJmtBLKWwzpO+BskrFOuRhPJHWecHFcQ3A7kFuNVZJkIcRlDi+yFg0c5M/LSR/Rpmkbeob2dHZYwhYXzl1aJiNjN5TmG+u8g4yE0HWykRwieoKnv3PjrUckWQhxmfKigfrWNuz/cT/P9nlWOrbrojKz8URS5eRw7oixTrlCWCfoNvliDaWgNtJqqAFJFkJUknc0kywdRmyzMyxNXYanqydj2o5xdlgCoPBspU7onUarwVRorPMJMVoNPaYZ/7boAR6+zo23gZFkIUQl+77YglZBxI7rxPPH3mZk9EiaekoHZ60rMxmjnysnh5xjxjoXNwjrDD1+fbFMRmC0tBocTJKFEFYWi4WUY5pAdYodwecoTCqUju3apDUcXQfb34ejG8BcZCz3CzNuI/W+3/i3eXdjsh9RqyRZCGF1/AejaGCXuEJeT5pPu4B2dGvWzdlhNXymYti/GLa9B6cPgm8z6HmvcTupZV9o2lJaDXWAJAshrA6uMooGuo0OJ2FzAv/X9/+kY9uRCk7DzjkQP8eoxBrWGcb9F7pMAjdPZ0cnLiPJQgjKiwYGE+mTzddZ2/Fy9eLOtnc6O6yGKesgbPsv7FsEZaUQMxIGPAqth0gLog6TZCEEcGD+BspcfYm5tRUzj73NqNajaOLRxNlhNRwWC6SsgW3vwtH14OYNPaZD/99BSIyzoxM2kGQhBHB4Xz6+upg97SwU7SiSjm17Kb0A+xYa/RFnk8C/OdzyIvS6v96X7G5sJFmIRi9rZ6JRNDAym9kpS2gf2J4uIV2cHVb9lncSdn4I8XONWkzNu8OEDyFuPLh5ODs6cQMcWjpRKTVKKXVYKZWilHquivWtlFJrlVL7lFLrlVKRldbNUEolW79mODJO0bjt/2o3Spfhc0cYiecSmRQ7STq2b1TmHlj6MLzVBTb9C1oNgvu/h4fXQ9dfSaKoxxzWslBKuQLvAiOAdGCnUmqF1jqh0mZvAJ9qrecppYYD/wCmK6WCgJeA3hh1gndZ981xVLyicTIXl3DsjC9h7lksL07G282b0W1GOzus+sVSBkk/wNb/wvHNxtzSfR6Afr81SmyIBsGRt6H6Aila66MASqmFwDigcrKIA560fr8OWGb9fiSwWmt9zrrvamAUsMCB8YpG6PCizZS6+9N2gCevHXub21vfjr+HFJezSUkB7Pnc6I/IOWaMh7jtZaPj2lvKuTc0jkwWEUBapdfpQL/LttkLTARmA3cB/kqp4KvsG+G4UEVjlbjlJB4mfw71LKRot3Rs2yQ33RhlvWueMQ9EZB+j07rjWHCVbtCGypH/s1Xd9NWXvX4aeEcpdR+wEcgAzDbui1LqYeBhgKioqJrEKhqh3CMZ1qKBZ3nv2Fd0DOpIp+BOzg6r7kqPh63vQsJy43XcWOj/mDG1qGjwHJks0oGWlV5HApmVN9BaZwITAJRSfsBErXWuUiodGHrZvusvP4HW+gPgA4DevXtfkUyEuJb9X2xBq2B8RwSSlJ7EC/1fkI7ty5WZIfFbI0mk7wDPJsbYiH6/hQD5gNaYODJZ7ARilFKtMVoM9wBTK2+glAoBzmmtLcD/AXOtq1YBryqlymdBv826Xgi7sFgsJKcqAtUpvvNMwMfNRzq2KyvOhd2fGbebck8YVV1HvWZUepUJgxolhyULrbVZKfU4xoXfFZirtT6olJoJxGutV2C0Hv6hlNIYt6Ees+57Tin1d4yEAzCzvLNbCHs4/kM8F9yD6NDhPP9OXcWdbe/E113mP+DcMSNB/DLfmGEuaiCM+ge0vx1cXJ0dnXAih/ZGaa1XAisvW/Zipe+XAEuusu9cLrY0hLCrAz8k4VoWxLEBeRQnFjfujm2t4cQ2oxRH4nfG3NSdJhj1mlr0cHZ0oo6QRxdEo1N0No+MIqNo4JxTy+kU3Im44Dhnh1X7ykxwcJmRJDJ/Aa8AGPQH6PsQNGnh7OhEHSPJQjQ6Bz9fT5mrH00GeZNyPoW/Dvirs0OqXUU5sOsT2P4B5GdCcDsY/SZ0myJTkYqrkmQhGp2k/YX46hJ+CNuNb4Yvt7e+3dkh1Y6zKbD9PdjzBZguGCXBx7wF7UaAi0Mr/4gGQJKFaFRO7Ugkxy2MThGn+c+JH7kr5i583BvwFJ1aQ+om49HXpFXg6g5d7jYefw2XYonCdpIsRKOyf+lulKUZmQPzKU0vbbgd2+YSOPCVUa8paz/4hMCQP0HvB8A/zNnRiXpIkoVoNMzFJaSe8SPMPYtP87+lS0gX2ge1d3ZY9lV41igLvvMjKMiCZh1h7H+gy6/A3cvZ0Yl6TJKFaDQOf7mJUnc/AnoUcDT3KDMHznR2SPZzOtE6VemXYC6GdrdC//eg7XCZqlTYhSQL0Wgc2noKT5M/a2L24nfKj5HRI50dUs1oDUfWGreajqwFNy/odg/0+x2EdnB2dKKBkWQhGoXcIxmc1mG0bXaK9zLWMCFmQv3t2DYVwb5FRkviTCL4hcGw56H3b8A32NnRiQbKpmShlPoKYzT199Y6TkLUK/usRQOz++ZRml/K3e3rYcd2fpbRFxE/By5kG08zjf8fdJ4Abp7Ojk40cLa2LN4D7gfeVkotBj7RWic6Liwh7MdisZCSqghSp/hC/UC3Zt2IDYx1dli2O7XfuNV0YIkx6jp2FAx4DKIHS3+EqDU2JQut9RpgjVKqKTAFWK2USgM+BOZrrU0OjFGIGjn+/U4uuAfRos0JUvNSeXnQy84OqXoWCyT/aJTiOLYR3H2g5wxjfERwW2dHJxohm/ssrDPYTQOmA78AnwODgRlcOveEEHXKgVXJuJYFsbHzQfxz/et2x3ZpoTHCevv/IDsF/FvArX+DXjPAO7D6/YVwEFv7LJYCHYDPgDFa65PWVV8qpeIdFZwQNVV0No/0ohAivLN4P3sNk9tPxsutDo43yMuEHR9A/MdQfB5a9ISJcyBunDHqWggns7Vl8Y7W+qeqVmite9sxHiHs6sD89Vhc/Sjofh6zxVz3Rmxn7Daeajr4NWgLdLjT6I9o2U/6I0SdYmuy6KiU2q21Pg9gncFuitb6v44LTYiaSzpgFA38vMkqevr2pG1AHbjfbymDwyuNek0ntoKHP/R92JiqNDDa2dEJUSVbk8VDWut3y19orXOUUg8BkixEnXVqRyLn3cJoFXqEE4VpPNLjd84NqCTfmIFu+/8gJ9WYw3rkq9BjOng1cW5sQlTD1mThopRSWmsNoJRyBTwcF5YQNbf/K6No4I6uiTQxNeG26NucE0hOKuz4EHZ/CiV5xi2mETOh/WhwlXGxon6w9Td1FbBIKfU/QAOPAD84LCohashcXELqWT9C3U/yYdFP3NP+Hjxda3HgWs5xSFgOCcsgYxcoV+g0Hvo/BpG9ai8OIezE1mTxLPBb4HeAAn4EPnJUUELUVOLCjZS6+1HaPrn2OrZzUo0EcXAZZO42loV3hVtehK6ToWmk42MQwkFsHZRnwRjF/Z5jwxHCPhK3ZeFh8ueL5mvoFdCLNgFtHHOinFQjOSQsM+axBmjeHW79q/HYa5CDzitELbN1nEUM8A8gDqh4SF1rLX8Jos45n5xBlg4nIvAox0vS+V3s4/Y9wbljRnI4uAxO7jGWtehhDJ6LGwdBre17PiHqAFtvQ30MvAT8GxiGUSdKHgIXddL+BVtABbO/02ECPAMY0WpEzQ+afeRigji1z1jWoqfRUR03Th55FQ2ercnCW2u91vpE1HHgr0qpTRgJRIg6w2KxkHJcEagy+cj1J37d9td4uN7gg3vZR4zBcgnLjGJ+ABG94baXjQQREGW/wIWo42xNFsVKKRcgWSn1OJABhFa3k1JqFDAbcAU+0lrPumx9FDAPCLBu85zWeqVSKho4BBy2brpNa/2IjbGKRix15Q4uuAfh2eIXzNrMpNhJ13eAsymQ8DUcXG7MXQ0Q2Qdue8WaIFraP2gh6gFbk8UfAB/g/wF/x7gVNeNaO1jHYrwLjADSgZ1KqRVa64RKmz0PLNJav6eUigNWAtHWdUe01t1t/UGEADi4KgXXsiAWt/mJvs36Et00uvqdziRdvMV0+qCxLLKvMWAubpw8xSQENiQL60X/V1rrZ4ACjP4KW/QFUrTWR63HWQiMAyonCw2UD11tCmTaeGwhrlB0Npf04hCaeaaRUnaCf8Y+cfWNzxy++BTTaeuvZMv+MGoWdBwLTSNqJ2gh6olqk4XWukwp1avyCG4bRQBplV6nA/0u2+avwI9KqScAX+DWSutaK6V+AfKA57XWmy4/gVLqYeBhgKgouX/c2B34bD0WV3+OxB4myCuIW6JuuXSD04kXWxBnDgEKovrDqNcgbiw0aeGUuIWoD2y9DfULsNw6S15h+UKt9dJr7FPV01KXJ5spGLPuvamUGgB8ppTqDJwEorTW2UqpXsAypVQnrXXeJQfT+gPgA4DevXtfTyITDVDSwQv4Wkr4sMlq7m03A3dXdzh96GIL4kwiRoIYALe/brQgmjR3dthC1Au2JosgIBsYXmmZBq6VLNKByr2BkVx5m+kBYBSA1nqrUsoLCNFanwZKrMt3KaWOALGAzJ0hqnRq2yHOu4URGPALZcrCpLOn4Z2+cPYwoKDVILjjDeg4BvzDnR2uEPWOrSO4be2nqGwnEKOUao3x9NQ9wNTLtjkB3AJ8opTqiDHg74xSqhlwznoLrA0QAxy9gRhEY6A1+77cgrJEsTx8Mf2Lioja9r6RIPo+ZLQg/MOcHaUQ9ZqtI7g/5spbSGitf3O1fbTWZutjtqswHoudq7U+qJSaCcRrrVcATwEfKqWetB7/Pq21VkrdDMxUSpmBMuARrfW56/3hRAOmNWQdgIPLMO9bQWreXwgq3UNCYCFvRk6EaU+BX7VPdwshbGTrbahvK33vBdyFDU8uaa1XYjwOW3nZi5W+TwAGVbHfV8BXNsYmGgutjcFx5Z3U546AciExfzIm9yZkRp8iyCuIYUNflqlIhbAzW29DXXLhVkotANY4JCIhKtMaTu41EkTCcjh31Cj33fomGPgEdLiTxKdW4WnOY07kWu5td5/RsS2EsKsbnXklBpBnVYVjaG0U6DtoTRA5x6wJ4mYY9HtjnmrfEADOJ6eTRTgBfvsxuZQxMXaik4MXomGytc8in0v7LE5hzHEhhH1obZT4Lm9B5KQaCaLNEBj8pDVBBF+x274vtoIKZnWbjQxsMZCW/lKOQwhHsPU2lL+jAxGNkNbGJEHl4yDOnwAXN2gzFG56GjqMBp+gq+5usVg4ckLRhHR2BaTw79jHai10IRobW1sWdwE/aa1zra8DgKFa62WODE40QFob04we/BoSVkBueYIYBkOehfZ3XDNBVJb67XYuuAdhCdpFiHcIQ1oOcXDwQjRetvZZvKS1/rr8hdb6vFLqJUCShaie1pAef/EWU24auLhD22Ew9DnocAd4B173YQ+sPoJrWRCfRf/I1HYzcHeRjm0hHMXWZOFSg31FY2SxQEb8xU7qvHRw9YC2w2HYX6D97eAdcMOHLzqbS0ZxCP5uyVzwNF1/KXIhxHWx9YIfr5T6F0bJcQ08AexyWFSifrJYIH2HkSAOrYC8DGuCuAVuecFIEF5N7XKq8qKB2yJ/ZlDEIFr4SRFAIRzJ1mTxBPAC8KX19Y8Yc1GIxs5igbTt1ltMKyA/E1w9od0tcMtL0H6U3RJEZUkHL+BjKWJDiwPMjn3b7scXQlzK1qehCoHnHByLqC8sFkjbdrEFkX/SSBAxIyBuJsSOBK8m1R/nBp3amsB5tzBcfNYT6hfGzZE3O+xcQgiDrU9DrQbu1lqft74OBBZqrUc6MjhRB+VlwryxkJ0Mbl7Q7lbodJeRIDxr5wnrfV/vQVlCWNz6RybGTMfNRbrPhHA0W//KQsoTBYDWOkcpJVXaGpuiHJg/EfJPwV0fGE8x1VKCKGe+UEzqOX/8OUp20yImxsiIbSFqg63JwqKUitJanwBQSkVTRRVa0YCZimDBFMhOgV8vMUZWO0Hil5swuflyJHgbN0XcRLivzE0hRG2wNVn8BdislNpgfX0z1ulMRSNQZoavHoQT22DSXKclCoBD27LwMPvyQ9tdzI79j9PiEKKxsbWD+welVG+MBLEHWA4UOTIwUUdoDd/9ERK/NaYi7TzBaaGcT07nNOG4e22lmX8YgyMGOy0WIRzBYtEUlpopKDFTUGwmr/ji9wUlJvKtr/OLy5eZyS8x0yrIh7+P7+zQ2Gzt4H4Q+D3G1Kh7gP7AVi6dZlU0ROtehd3z4KanoN9vnRrKvi+2gArhu1ZrmRgzWTq2RZ1RVn6RL7ZeyCtd2Msv6nnFFy/6FRf8yy78BSVmm87n4+GKn6cb/l5u+Hm5Y9GO7xWw9a/t90AfYJvWephSqgPwN8eFJeqEHR/CxtehxzQY/oJTQ7GYy0g54YqfTiU1PIe7Yu5yajyiYSizaOsF21RxYc+vdIHPLzZdsezihf/iPoWlZTadz8/Tzfjycqu42Ic38TIu+p7u+Hm54V9pvZ+XG00qrSvf39VFOfiduZKtyaJYa12slEIp5am1TlRKtXdoZMK5Di6Dlc9A7O1w52xQtf/LWdmx73ZQ5B7IWd8fuTnyZunYbuTMZZZLPp1XvtCXX+AvWV/p4l/5wn/Bhou8UuDncekFvqm3O5EB3ldc+P0vu7D7V7ro+3m44eKEi7y92Jos0q2VZpcBq5VSOdgwraqop45tgqUPQcu+Roe2q/Nv9xxck4KrOYjvYnfwRuxsZ4cjbpCpzHLlhdx6y+bKC/uVt3PKL/RFJhsv8p5uNPFyr7hgB/h4EBnkY3x693TD36vqT/P+ldb5uLvW64u8vdjawV3e5v+rUmod0BT4wWFRCec5uQ8WToWgNjBlIXj4ODsiLpzOIaO4Ge5qPwHBzRiU2erSAAAgAElEQVTU4opp24WDlZotlTpdTZde1C/vgK3ito2RDEyUmC3VnstFYVyoK30yD/L1oFWw76Wf1qu4sFde7+PhinJyi7ghue6PjFrrDdVvJeqlnFT4fJIx0G7aVzbPK+FoB+ZvxOLqz8bwdUyMmYiri6uzQ6o3ik1lVXxav/5P86U2XORdXdQlF/ImXu408/ekdYjvxU/v5Rf4y5JB5U/33u5yka+LnH9/QdQNBWfgs7vAXAK/WQFNI50dUYWkhAt4lxWyLzqDNxpBx7bWmhKz5dIL9mX32q+4sFfxab6g2ExpWfUXeXdXVXHxLr9whzfxsl7UjXvwVX2av/yi7+nmIhf5BkyShYCSAvjibsg7Cfcuh9AOzo6oQuaWg+S6hVHg9Q1DWw0j1Kf+VZnZfSKHo2cKKSiu9Jx85Qt75Vs71tc+7oon+gXSKsAdhXEBdgGaWL/wBOVlLFNK4aLcUModF1X++ir/UvXya9OAyfplZX1ZnA/FwFl7v2nC7ry8vIiMjMTd/cYmCXNoslBKjQJmA67AR1rrWZetjwLmAQHWbZ7TWq+0rvs/4AGgDPh/WutVjoy10TKXwqLpRl/FPZ9DVD9nR3SJA1/vRVlC+KH9Fl6O/Zezw7GZ1pqtR7N5a00yO46du2Sdp5uL9cmZi5/mW5Z3ulo/wfcJKaNVeBABgUG4ubjg4qJwVVj/Vbi4KFzkU7ywkdaa7Oxs0tPTad269Q0dw2HJQinlijFZ0gggHdiplFqhtU6otNnzwCKt9XtKqThgJRBt/f4eoBPQAlijlIrVWtv2MLOwjcUCyx+FIz/BuHeNyYnqEPOFYlJz/PG0JOLZIpABLQY4O6Rqaa3ZesSaJFLPEdbEk7+OiWN4hzD8vdzw9XTDw62qiScvdejQIVpHhMttHWEXSimCg4M5c+bMDR/DkS2LvkCK1voogFJqITAOqJwsNNZWNcYTVuWP447DKIFeAhxTSqVYj7fVgfE2LlrDj8/D/sVwy4vGwLs65tBCo2jg/qabmBQ7CRdV/UXWWbTW/JySzey1SexMzSG8iRd/G9uJyX1a4uV+Yx3ykiiEPdX098mRf30RQFql1+nWZZX9FZimlErHaFU8cR37iprY8jZsexf6PQKD/+jsaKqUuD0Lj9LzbOmQwvh2450dTpW01mxKPsOk/21l2pztpJ0rYua4Tqx/ZigzBkbfcKKoS/r160f37t2JioqiWbNmdO/ene7du5Oamnpdx1m6dCmJiYnXff7BgwezZ8+e696v3BtvvMEXX3xxw/vXhrvvvpujR49Wue6HH36gZ8+edOnShV69erF+/foqt8vOzuaWW24hJiaGkSNHkpuba9cYHZksqkpjlxcwmQJ8orWOBO4APlNKudi4L0qph5VS8Uqp+Jo0rxqdPQtg9YvQaQKM/IfTR2dXJedwGqcJp9B9J0NaDyPEO8TZIV1Ca83GpDNMfG8L0+fsIPN8EX8f35kNfxrKvQMaRpIot337dvbs2cPMmTOZPHkye/bsYc+ePURHR1/XcW40WdSEyWTis88+Y/LkybV63uv1yCOP8M9//rPKdaGhoXz33Xfs37+fuXPnMn369Cq3e+WVV7j99ttJTk7mpptu4vXXX7drjI5MFulAy0qvI7ly1PcDwCIArfVWwAsIsXFftNYfaK17a617N2vWzI6hN2BJP8Lyx6D1ELjrf+BSN2/t7F+wFZQLa9tuYVLsJGeHU0FrzfrDp5nw3hbunbuDU7nFvDy+M+ufGcr0/q3wdGs4ScIW33//PQMGDKBnz55MnjyZwsJCAJ555hni4uLo2rUrzz77LJs2bWLlypU8+eSTN9QqKTd//ny6dOlC586d+fOf/1yx/P333yc2NpahQ4fy4IMP8oc//AGA1atX06dPH1xdjf+Xbdu20bVrVwYOHMgzzzxD9+7dAThy5Ag33XQTPXr0oFevXmzfvh2ANWvWMGzYMCZNmkRMTAzPP/88n376KX369KFr164VP8e0adN47LHHGDZsGG3btmXjxo3MmDGDDh068MADD1TE+fDDD9O7d286derEzJkzK5YPHTqUH374gbKyK7tle/bsSfPmzQHo0qULBQUFmEymK7Zbvnw5M2bMAGDGjBksW7bsht7jq3Fkn8VOIEYp1RrIwOiwnnrZNieAW4BPlFIdMZLFGWAF8IVS6l8YHdwxwA4Hxto4pMfD4hkQ1gkmzwc3T2dHVCWLuYyUNFe8LEcwt/Whf/P+zg7JSBJJZ5i9Jpk9aeeJCPDmlbs6M6lXpMMTxN++OUhCZp5djxnXogkvjelUo2OcPn2aWbNmsXbtWnx8fHjllVeYPXs2DzzwACtXruTgwYMopTh//jwBAQHccccdTJo0ifHjb+yWYnp6Os8//zzx8fE0bdqUW2+9lW+//ZZu3boxa9Ysdu/eja+vL0OHDqVv374A/Pzzz/Tq1aviGPfffz/z5s2jb9++PP300xXLmzdvzurVq/Hy8iIxMZEZM2ZUJIy9e/dy6NAhmjZtSnR0NI8++ig7d+7kzTff5J133uGNN94AIDc3l3Xr1vHVV18xZswYtm7dSocOHejZsycHDhygc+fOzJo1i6CgIMxmc0USiouLw9XVlejoaA4cOEC3bt2u+h4sWrSIfv36Vfn4a3Z2NuUfmiMiIjh58uQNvc9X47BkobU2K6UeB1ZhPBY7V2t9UCk1E4jXWq8AngI+VEo9iXGb6T6ttQYOKqUWYXSGm4HH5EmoGjqTBJ/fDX5hxuhsrybV7+MkR7/dTpF7IOleK5zesW20JM7w1tpk9lqTxD8mdGFiz0ibnmpqyLZs2UJCQgIDBw4EoLS0lMGDBxMUFISLiwsPPfQQo0eP5s4777TL+bZv387w4cMJCTFuSU6dOpWNGzdSXFzM8OHDCQwMBGDSpEmcOHECgJMnT9KjRw8Azp49S2lpaUUimTp1KmvWrAGgpKSExx9/nL179+Lm5saRI0cqztuvXz/CwsIAaNOmDSNHjgSMT/lbt1585mbMmDEVy1u0aEFcXBwAcXFxpKam0rlzZxYsWMCcOXMwm81kZmaSkJBQsV1oaCiZmZlXTRb79+/n+eefZ/Xq1Ta9X/Z+QMKh4yysYyZWXrbsxUrfJwBVFvrRWr8CvOLI+BqNvEyYPwFcXGH6UvCr2wPbDq49gqs5iPWdE3i6nXPGVmit+SnxNLPXJrMvPZfIQG9mTejCBCckiZq2ABxFa82oUaP47LPPrlgXHx/P6tWrWbhwIe+99x4//vjjVY9T+QI+YcIEXnzxxSq301eZs+FqywG8vb0pLi6udrs333yTli1bMn/+fEwmE35+fhXrPD0vtsBdXFwqXru4uGA2m6/YrvI2lbdLTk5m9uzZ7Nixg4CAAKZNm1YRG0BxcTHe3t4sWbKEl19+GYBPPvmE7t27c+LECSZMmMD8+fOvOk6i/NHYZs2akZGRQXi4fSszN+6PRo1B0XmYPxGKcoy5s4PaODuia7pwOofMklDK9G4GxQwh2Du4Vs+vtWZNQhZj3/mZB+bFk3OhlNcmdmHd00O5p29Uo29NVDZw4EA2bNhQ8RRPYWEhycnJ5Ofnk5eXx5133sm///1vfvnlFwD8/f3Jz8+/4jgeHh4VneZXSxQA/fv3Z926dWRnZ2M2m1m4cCFDhgyhX79+rFu3jvPnz2MymVi6dGnFPh07diQlJQWAZs2a4e7uTnx8PAALFy6s2C43N5fmzZujlGLevHnXTCw3Ki8vD39/f5o0acLJkydZterSccbJycl06tSJSZMmVbwf3bt3Jycnh9GjR/PGG2/Qv//Vb8mOHTuWefPmATBv3jzGjRtn1/jlN78hMxXBgilwNtkYnd2iu7MjqtaBzzZgcXFnW9QW7o69u9bOq7VmdUIWY97ZzIOfxpNbZOL1iV356amhTO4Thbur/KlcLiwsjDlz5jB58mS6devGwIEDSUpKIjc3l9GjR9OtWzeGDx/Ov/5ltA6nTJnCq6++esMd3JGRkcycOZOhQ4fSvXt3+vfvz+jRo4mKiuKZZ56hb9++3HbbbXTq1ImmTZsCcMcdd7Bhw8Xap3PnzuX+++9n4MCBuLi4VGz3+OOP89FHH9G/f3+OHz9+ScvAXnr27ElcXBydO3fmoYceYtCgizdVMjMzadq0KVU9qDN79myOHTvGSy+9VPHYcnZ2NmD0wZQ/VvznP/+Z7777jpiYGDZu3Mgzzzxj1/iVIzKoM/Tu3VuXf2IQQJnZ6MxO/A4mzYHOE50dkU3mP7iQ4jIzy+5cwDcTvnX4wDStNT8mZPH22mQOZubRKtiHx4e1Y3yPCKcmiEOHDtGxY0ennb++KSgowM/PD5PJxLhx4/jd735X0YcwduxY3nrrLdq0aVOxHRiPmp47d44333zTmaED8M9//pPQ0NCKp5kcparfK6XULq117+r2lUKCDZHWsPIpSPwWbn+93iSKzJ8PkusWyjmPJUxqf7dDE4XFcjFJJJzMIzrYhzfu7sb47i1wk1ZEvfPCCy+wfv16iouLGTVq1CWd6q+99hqZmZm0adOGFStW8Prrr2M2m4mOjuaTTz5xXtCVBAcHM21a3auiUJm0LBqida/ChteMkdm3vuTsaGy26pnPOZLbjKXdZrL4gW8I9Aq0+zmMJHGKt9Ykk3gqn9Yhvjw+rB3j6liSkJaFcARpWYiLdn5kJIoe04yaT/WEqbCY4zlNcDXvp2eXgXZPFBaLZtXBU8xeaySJNiG+/HtyN8Z0rVtJQoi6SpJFQ5KwHL57GmJHwZ2z62QZj6tJXLgRk5svh4K3MiP2Wbsd12LRfH/gFG+vTeZwVj5tmvny1uTujOnWAleZV1kIm0myaCiObYKvHoTIPjDpY3CtX/+1h3acxt3kzYk+JfQOq7ZFXC2LRbPywEneXptMUlYBbZv5Mvue7tzZVZKEEDeifl1RRNVO7YeFUyGwNUz9Ejx8nB3RdclJPMEZwilyXcXEjpNq1LFdZtGs3G8kieTTBbQL9ePtKT0Y3aW5JAkhakBu1tZ3OanGoDtPf2N0tk+QsyO6bvsWGkUDt7Tfzbi2NzaQqMyiWb4ng5FvbeSJBcYgsP9M6cGqP9zMWLnlVCNSotzxrlWi/PTp0wwdOhRfX9+KAolVqc8lyoWjFZ6FzyaAucSo99Q00tkRXTeLuYwjaW54FCUR268PAV4B17V/eZK47d8b+P3CPbgoeGeqkSSkX8I+pES5412rRHl5kcbXXnvtmseozyXKhSOVFBiFAfMyYOoiCK2fj1ke/cYoGngkcOt1jdgus2iW/ZLBCGuScHNx4d2pPfnh9zdzZ9cWuEiSqBVSotz4ORxZotzPz49Bgwbh5eV1zfemPpcoF45iLoVF0+HkXqOMR1Q/Z0d0ww6uPYqrOYAj/XPpGdqz2u3NZRZW7M3knZ9SOHq2kA7h/rz3656M7BTecBPE988Z/VL2FN4Fbp9Vo0NIifLaL1F+LfW2RLlwEIvFmLzoyE8w9j/Q/nZnR3TDLmSdI7MkBFPZNsZ1vXbHtrnMwvI9mbyzLoVj1iTxv2k9uS2uASeJOk5KlNduifLrVa9KlAsHWP0C7F8Ew1+Anvc6O5oaOTB/IxbXJuxutYvZbZ+schtzmYVlezJ556dkUrMvENe8Cf+b1ovb4sIaT5KoYQvAUaREee2VKLeFlCgXF/38Nmx9B/r+Fm56ytnR1NjhhCI8ijOIuLkLTT2bXrLOVGZhUXwat/xrA08v3ouPhxsfTO/Fd/9vMKM6S2uiLpAS5dfnRkuU28rRJcqlZVFf7F1otCo63QWjZtWr0dlVydx8gDz3MM55LuHuDg9XLDeVWfh6dwbvrEvhxLkLdGrRhA/v7c2tHUMdXoFWXJ/KJcpLS0sBePXVV/H29mbChAmUlJRgsVguKVH+29/+ljfffJNly5Zd99NUlUuUa60ZM2YMo0ePBqgoUR4REXFFifLKHczlJcr9/f25+eabLylRPmnSJBYsWMCtt97q8BLlbdq0sblEefnPfuHCBUwmE0uWLGHt2rW0b9+e+++/n9///vd0796dP//5z/zqV7/i/fffp3Xr1nz55Zd2jV8KCdYHyathwT3QaqAxgVEdnTv7ehhFA0PYNmgO8+79ErNFs3R3Ou+sSyHtXBFdIpry+1tiuKWRJgkpJHh9pES5baSQYEOWHg+L7oXQOJj8eYNIFKbCYlJzmqDM+xjRexxf7kzjnXUppOcU0TWyKX8b24lh7RtnkhA3RkqUO560LOqys8kw5zbwagK/+RH8w5wdkV3sn7OKjTvdORz4PvEB08nMUXSLbMofbo1laPtmkiSQloVwDGlZNER5mfDZXeDiCtOWNphEUWq2sOfnU7jjy9pm3sR4BvLK+BiGxkqSEKIuk2RRFxWdN+o9FeXAfd9CcFtnR1RjJeYyFsen8/XyXQx3j6DIsoqnb76P+3oNlCQhRD0gyaKuMRXBginGLahfL4YWPZwdUY2UmMtYtDON/64/wsncYp46cxbcAzjaK5Oneg2RRCFEPSHJoi6xlBlzUpzYAhPnQNthzo7ohhWbylgUn8Z71iTRq1Ugr93VieTXiykrSuTmoWMkUQhRjzh0UJ5SapRS6rBSKkUp9VwV6/+tlNpj/UpSSp2vtK6s0roVjoyzTtAavvsjJH4Lo16DLpOcHdENKTaVMW9LKkP/uZ4Xlx8kMtCb+Q/0Y8kjA2hx6AjFHoEcD45ndJvRzg5V2EhKlDve5SXKd+7cSefOnWnXrh1PPll1dQOtNY8++ijt2rWjW7duNXqPbOGwloVSyhV4FxgBpAM7lVIrtNYJ5dtorZ+stP0TQOV7LkVaa9uHL9Z362fBrk9g8JPQ/xFnR3Pdik1lLNhxgv9tOEJWXgl9o4P416+6MaBtcEULYv+aI7iag/C8IwJ/D38nRyxsVV5Q75NPPiE+Pp533nnnho6zdOlSXFxc6NChgz3Du6byEuW7d++utXPeiPIS5e+9917F648//pjevXszcuRIVq9ezYgRIy7Z55tvviEtLY2UlBQ2b97MY489xs8//+ywGB3ZsugLpGitj2qtS4GFwLXGn08BFjgwnrpr5xzYMAu6T4NbXnJ2NNel2FTG3M3HuPn1dfztmwRaBfvyxUP9+PK3/RnYLqQiUVw4mc1JUxillngmdK/bcwsI20mJcuPnsGeJ8rS0NIqLi+nTpw9KKaZPn15lufHly5dz771GfbjBgwdz6tQpzpw5c0Pvqy0c2WcRAaRVep0OVFlLWynVCmgN/FRpsZdSKh4wA7O01le8W0qph4GHAaKiouwUdi1LWA7fPQUxI2HM7HpTxqPYVMbn242WxJn8Evq1DmL2PT0Y0Da4yu33z9+IdmlKeqc0OgV3quVo67fXdrxG4jn7ThrUIagDz/Z9tkbHkBLljilRXlRURMuWLStii4yMJCMj44r3IyMjo8rtrlYypKYcmSyquupdbQTgPcASrXXlmT+itNaZSqk2wE9Kqf1a6yOVd9JafwB8AMagPHsEXatSNxsd2pG94e5PwLXuP2+QX2xi4Y403t94lLMFJQxoE8x/pvSgf5uqk0S5xENFeFjy6XXHbdKx3UBIiXLHlCgPCLhytsiq/maqGlDtyL8tR16d0oGWlV5HAplX2fYe4LHKC7TWmdZ/jyql1mP0Zxy5ctd66tR+4xHZwNbGTHcePs6O6JrSzl3gky2pfLkzjYISMwPbBvPu1B70qyZJAGRs2k+BRzi5bl8ztV3dLLddl9W0BeAoUqLcMSXKIyMjSUu7eFMmPT2dFi1aXBFz+Xb9+/e/5nb24sg+i51AjFKqtVLKAyMhXPFUk1KqPRAIbK20LFAp5Wn9PgQYBCRcvm+9lXPcGHTn4QfTl4JPkLMjqpLWmvjUc/xu/i6G/HMd87akckvHUFY8PogvHupvU6IA2LvsF5TFhMuoIHzdfR0ctagtUqL8+thaorxly5Z4enqyc+dOtNZ89tlnVZYbHzt2LJ9++ikAmzdvJiwszGG3oMCBLQuttVkp9TiwCnAF5mqtDyqlZgLxWuvyxDEFWKgv/d/pCLyvlLJgJLRZlZ+iqtcKz8L8CWAuht+sgqaRzo7oCqYyC98fOMWcTUfZm55LU293Hr65LTMGtqJ5U+/rO1ZhMWm5gSjzPsYOkI7thkRKlF+f6ylR/t5773HfffdRXFzMnXfeWfEk1LvvvounpycPPvggY8aM4fvvv6dt27b4+vpWzGXhKFJIsDaVFMC8MXA6Ae5dDlH9nR3RJXIvmFiw8wTztqRyMreY1iG+/GZQNBN7ReLjcWOfK/Z9+AObdnlwtsUSXnrxv3aOuOGSQoLXR0qU20YKCdYHZSaj1PjJPUap8TqUKFLPFvLxz8dYvCudC6VlDGgTzMvjOzOsfWiNZ6Q7sD0L9zJfOk2sv6PRRd0nJcodT1oWtcFigWWPwL4vYczb0Muxnx5sobVm29FzzNl8jLWJWbi5KMZ2i+A3g6Pp1KJp9QewQc6h43zxVjIlrOaRd17Cx71ud+LXJdKyEI4gLYu6bvULRqIY/rzTE0Wp2cK3+zKZs/kYBzPzCPL14Ilh7Zg2oBWh/l52PdfuzzcDYTDcSxKFEPWcJAtH+/lt2PoO9H0Ybnq6+u0d5FxhKV9sP86nW49zOr+EdqF+/GNCF+7qEYGXu6vdz2cxl3Es0xP3siRuGTHF7scXQtQuSRaOtHeh0aqIGw+jZjlldHbK6XzmbE5l6e50SswWbooJ4fVJXRni4MmGjizfQolHEIWBP9Mh6FGHnUcIUTskWThK8hpY/hi0vhkmfGDMeFdLtNZsTjnLnM3HWH/4DB5uLkzoEcFvBrcmNqx2Cvj9siYJV3MYMfcMqJXzCSEcy6Elyhut9F2waDqEdjSefHKz/zPbVSk2lfHlzhOMemsT0+fs4EBGHn8cEcvW54Yza2LXWksUF05mc7YsErPexcg4+5R6EM4jJcod7/IS5c899xyRkZFVlv6o7OWXX6Zdu3Z06NChonSJo0jLwt7OJsMXd4NvM/j1V+DVxOGnPJNfwvxtx5m/7TjZhaV0CPfnn5O6MrZ7Czzdaq9FU27XvLVolxDoi3RsNwBSotzxLi9RPm7cOB5//HE6d+581X327dvH0qVLSUhIIC0tjVGjRnH48GFcXBzTBpCWhT3lnYTPJgAKpn8N/mEOPV3iqTyeWbyXQbN+YvbaZLq3DOCLB/vx/e9v4u7eLZ2SKABSkkx4FKdx20Tp2G7opES58XPYs0Q5wIABAwgPD7/me7F8+XKmTJmCh4cHbdu2JSoqil27dt3Q+2oLaVnYS9F5o95T0Tm471sIbuuQ01gsmg1JZ5iz+RibU87i5e7Cr/pEcv+g1rRt5lf9ARwsfcNeLng0p8z7R2IDnT+epCE49eqrlByyb4lyz44dCK90sb0RUqLcMSXKu3XrZtP7kZGRwdChQytel5co79Onzw29v9WRZGEPpmJYOBXOJsGvF0GLHtXvc52KSsv4anc6c38+xtEzhYQ18eRPo9oztW8UAT4edj/fjdqxdAfKEkXUrxrPJIeNlZQod0yJcluTRUMqUd44WMpg6YNw/GeYOAfaDrfr4bPyivl0ayqfbz/B+QsmukQ05a3J3bmjS3M83OrWXcTSgiLOFIbjUrafUX0ed3Y4DUZNWwCOIiXKHVOi3Fa2ljK3l7p1talvtDZmuTv0jTGOosskux36QEYuT365h8Gv/cR/1x+hX+sgFv12ACseH8T4HhF1LlEA7Jn/I2Y3X1y6FuHlZt/R4KLukRLl18fWEuW2Gjt2LAsWLKC0tJQjR45w/PjxS2652Zu0LGpiw2uw62MY9Afo/7saH67Moll7KIs5m4+x/dg5fD1cmda/FfcPbE1UcN1/qihxZzbuFhPDpksp8sZASpRfn+spUf7HP/6RRYsWkZeXR2RkJI888gjPP/88X3/9Nfv37+fFF1+kW7dujB8/no4dO+Lm5sZ///tfhz0JBVJI8MbtnAPf/RG6TYXx/63R6OzCEjOL49P4eEsqx7MvEBHgzX0Do5nctyVNvNztGLTjZB84xsL/HMHFbTO/e/evzg6n3pNCgtdHSpTbRgoJ1raEFbDyaYi5Dca+fcOJIvN8EfO2pPLFjhPkF5vpGRXAn0Z2YGSnMNxc695tpmv5ef4aoDUR42OcHYpohKREueNJy+J6pW42xlI072pMYORx/dOE/nIihzmbj/H9gVMAjOoczgODW9MzKtDe0dYKi7mMOQ9/DWWnmT7nN9JfYQfSshCOIC2L2nLqACyYCoGtYOqi60oU5jILPyZk8dGmo+w+cR5/LzceGNyaGQOjiQi4vqlK65r9i9dQ6hGEd0SiJAohGihJFrbKOW4MuvPwhWlLwSfIpt3yik0s2pnGxz+nknG+iKggH14aE8fdvVvi59kw3v79647haolg8P0TnR2KEMJBGsbVytEKs2H+BDAXwf0/QEDLanc5kX2Bj7ccY3F8OgUlZvq2DuLFMXHc2jEM1xpOVVqX5GecJo/WuLv8QmzkGGeHI4RwEEkW1SkpMAoD5qbD9GUQFnfVTbXWxB/PYc6mY/yYcAoXpbiza3MeGNyGLpH2maq0rtn40Qq0SxvCb7l2HRshRP1Wvx65qW1lJlh0L2T+ApPmQquq52YwlVlYvieDce/+zN3/28rWo9k8MqQtm58dzlv39GiwiQLgVKonnsVpjJww1dmhCAeREuWOV7lEeX5+PnfccQft27enU6dO/OUvf7nqflKivC6wWIzJi46shTFvQ4fRV2ySe8HEFztO8OnWVE7mFtMmxJe/j+/MxJ4R+Hg0/Lf20OrNFHtG4Be4Aw/XulOfStiXlCh3vMolypVSPPvsswwZMoSSkhKGDRvG6tWrGTFixCX7NKgS5UqpUUqpw0qpFKXUc1Ws/7dSao/1K0kpdb7SuhlKqWTrV+2XL13zIuz7EoY9D70uPf3RMwW8sEEsJ6sAAArpSURBVOwA/f+xltd+SKRNM1/m3tebNX8cwvT+rRpFogDYvewXlMXEgPtvd3YowkmkRLnxc9izRLmfnx9DhgwBjHpTPXr0ID09/Yr3osGUKFdKuQLvAiOAdGCnUmqF1jqhfBut9ZOVtn8C6GH9Pgh4CegNaGCXdd8cR8V7iS3/Mb76PAQ3P10eK1uPZjN38zHWJp7G3cWFsd1b8JtBrYlr4fgJjuqa4vxC8kui8dSHiI0d6exwGrRNi5I4m1Zg12OGtPTjpl/F1ugYUqLc8SXKc3JyWLlyJX/605+ueD8aUonyvkCK1voogFJqITAOSLjK9lMwEgTASGC11vqcdd/VwChggQPjNez9En58HuLGwe2vUVJm4Zu9J5m7+RgJJ/MI9vXgieExTOsfRah/4x1TsP7DRZS5tSKyl/Pn0BDOISXKHVui3GQyMXnyZJ566ilatWp1xfvRkEqURwBplV6nA/2q2lAp1QpoDfx0jX0jHBDjpVLWwPJHIfomzo18l8/XHeXTbcc5k19CbJjf/2/v3mPkKss4jn9/vS6BCqXlUlmQogXLJeByp0hQ1JCqVAFTxIRLS5BIk/KXUjFR4Q9Tif5hvIGIASVAKBcrlAByscGkhRbbbmu5tKTg0lKWNRYJFHp5/OO8C+N0Zs/M7J6ZYff3STZ7duc9Z57zzuw+c95zzvOy8ILjmHXCIXSMbc0MdO1kS/dOxqqPL17S3iUKhoPBHgEUxSXKiytRHhHMnTuXY489lnnzKpf7H04lyiuluGqv1kXAoojYVc+6kq6UtELSit7e3gbDTHpWwt2X8N7EI/nR3t/n9Buf5mePvcjRUz7G7XNO4ZFrzmL2yYc5UQAbV/6Dd8ZNZa99X2H8+JF7dDXSuUR5feopUb5gwQK2b9/+wRBXJcOpRHkPUHr3WiewuUrbi4Cry9Y9u2zdp8pXioibgZshqw3VaKDx5kvs+OMF/Gf3BL782tW8tXUb53d1MmfG4Uw7aEKjmx22lv/pKeA4Trz4s60OxVrIJcrrU2uJ8k2bNrFw4UKmT59OV1cXAPPnz+fyyy8fniXKJY0BXgTOAV4DngUujoh1Ze2OAh4BpkYKJp3gXgl0pWbPASf2n8OopNFCgmuW/51nfrMJ+PDQRRUPbKzfjrETGb/zVeb84dutDmXYciHB+rhEeW3aspBgROyUNI8sEYwGbo2IdZKuB1ZExOLU9JvAXVGStSLi35JuIEswANcPlCgG44BJE1Esg1FjGD3KQ0y1GL27jyO/uucJN7NWcYny4rlEuVkb8pGFFWEwRxYu92FmZrmcLMza1HA56rf2MNj3k5OFWRvq6Oigr6/PCcOGRETQ19dHR0fjl7qPjCJGZh8xnZ2d9PT0MOj7h8ySjo4OOjs7G17fycKsDY0dO5apU6e2OgyzD3gYyszMcjlZmJlZLicLMzPLNWxuypPUC7wyiE1MBt4conCGkuOqj+Oqj+Oqz3CM6xMRcUBeo2GTLAZL0opa7mJsNsdVH8dVH8dVn5Ecl4ehzMwsl5OFmZnlcrL40M2tDqAKx1Ufx1Ufx1WfERuXz1mYmVkuH1mYmVmuEZssJN0o6XlJayTdL2m/Ku3OlfSCpA2Srm1CXN+QtE7SbklVr26QtElSt6RVkgqfyKOOuJrdX/tLekzSS+n7xCrtdqW+WiVpcaU2QxTPgPsvabyku9PjyyUdXlQsdcZ1maTekj66ogkx3SrpDUlrqzwuSb9IMa+R1FWpXQviOlvStpK+qj5x+NDGdaikJyWtT3+L8yu0Ka7PImJEfgFfAsak5YXAwgptRgMbgSOAccBq4OiC45oOHEU25/hJA7TbBExuYn/lxtWi/vopcG1avrbS65gee7sJfZS7/8B3gN+m5YuAu9skrsuAXzbr/ZSe8yyyqZPXVnl8JvAwIOA0YHmbxHU28GAz+yo97xSgKy1PIJu2uvx1LKzPRuyRRUQ8GhE704/LgErlGE8BNkTEyxHxPnAXMKvguNZHxAtFPkcjaoyr6f2Vtn9bWr4N+FrBzzeQWva/NN5FwDmSip70vRWvS66IWAoMNF3yLOD2yCwD9pM0pQ3iaomI2BIRz6Xl/wLrgUPKmhXWZyM2WZSZQ5aNyx0C/Kvk5x72fHFaJYBHJa2UdGWrg0la0V8HRcQWyP6YgAOrtOuQtELSMklFJZRa9v+DNunDyjZgUkHx1BMXwAVp6GKRpEMLjqkW7fz3d7qk1ZIelnRMs588DV9+Blhe9lBhfTasS5RL+itwcIWHrouIP6c21wE7gTsqbaLC7wZ9+VgtcdVgRkRslnQg8Jik59MnolbG1fT+qmMzh6X+OgJ4QlJ3RGwcbGxlatn/QvooRy3P+Rfgzoh4T9JVZEc/ny84rjyt6KtaPEdWIuNtSTOBB4BpzXpySfsA9wLXRMRb5Q9XWGVI+mxYJ4uI+MJAj0u6FPgKcE6kAb8yPUDpJ6xOYHPRcdW4jc3p+xuS7icbahhUshiCuJreX5K2SpoSEVvS4fYbVbbR318vS3qK7FPZUCeLWva/v02PpDHAvhQ/5JEbV0T0lfz4O7LzeK1WyPtpsEr/QUfEEkm/ljQ5IgqvGSVpLFmiuCMi7qvQpLA+G7HDUJLOBb4HnBcR71Rp9iwwTdJUSePITkgWdiVNrSTtLWlC/zLZyfqKV240WSv6azFwaVq+FNjjCEjSREnj0/JkYAbwzwJiqWX/S+O9EHiiygeVpsZVNq59Htl4eKstBi5JV/icBmzrH3JsJUkH959nknQK2f/RvoHXGpLnFfB7YH1E/LxKs+L6rNln9NvlC9hANra3Kn31X6HycWBJSbuZZFcdbCQbjik6rq+TfTp4D9gKPFIeF9lVLavT17p2iatF/TUJeBx4KX3fP/3+JOCWtHwG0J36qxuYW2A8e+w/cD3ZhxKADuCe9P57Bjii6D6qMa6fpPfSauBJ4NNNiOlOYAuwI7235gJXAVelxwX8KsXczQBXBzY5rnklfbUMOKNJcZ1JNqS0puT/1sxm9Znv4DYzs1wjdhjKzMxq52RhZma5nCzMzCyXk4WZmeVysjAzs1xOFmZ1kPT2INdflO4iR9I+km6StDFVEV0q6VRJ49LysL5p1j5anCzMmiTVEBodES+nX91Cdvf2tIg4hqzy6+TIiv09DsxuSaBmFThZmDUg3SF7o6S1yuYVmZ1+PyqVf1gn6UFJSyRdmFb7FukOc0mfBE4FfhARuyErRRIRD6W2D6T2Zm3Bh7lmjTkfOAE4HpgMPCtpKVkpkcOB48gq4K4Hbk3rzCC7OxjgGGBVROyqsv21wMmFRG7WAB9ZmDXmTLIqrbsiYivwN7J/7mcC90TE7oh4nax0Rr8pQG8tG09J5P3+GmBmreZkYdaYahMWDTSR0btktaEgqy10vKSB/gbHA9sbiM1syDlZmDVmKTBb0mhJB5BNxfkM8DTZJEKjJB1ENgVnv/XApwAim0tjBfDjkgqm0yTNSsuTgN6I2NGsHTIbiJOFWWPuJ6v+uRp4AvhuGna6l6xS6VrgJrKZzLaldR7i/5PHFWSTOm2Q1E02j0T/3AOfA5YUuwtmtXPVWbMhJmmfyGZRm0R2tDEjIl6XtBfZOYwZA5zY7t/GfcCCaMP52G1k8tVQZkPvQUn7AeOAG9IRBxHxrqQfks2J/Gq1ldMERQ84UVg78ZGFmZnl8jkLMzPL5WRhZma5nCzMzCyXk4WZmeVysjAzs1xOFmZmlut/Rih7DzL3hVoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    pyplot.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "    print(x_axis,'^^^^', np.array(accuracy_s1[:,j]),'^^^^', gamma)\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuracy' )\n",
    "pyplot.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gamma为1.0/10/100的时候，均有C值使准确率为1。取准确率1时gamma的最小值1，然后C取1得最优解"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "整体来看，SVM的复杂度最高，然后是svc，LR的复杂度相对低很多，但是准确率不高。RBF核的SVM准确率可以达到1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从算法层面来看。LR使用logistic损失，即误差平方和损失，SVM采用Hinge损失。LR对所有特征计算权重，对异常值敏感，但是SVM只在乎支持向量，容易忽略异常值，对异常值不敏感。\n",
    "LR训练较快，但是准确度较低；SVM错误率低，但是计算量较大，适用于数量比较小的样本数据。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
